BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 36.4 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "os"
13 "os/exec"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "gitlab.com/Timfa/bluegit/internal/access"
21 "gitlab.com/Timfa/bluegit/internal/ci"
22 "gitlab.com/Timfa/bluegit/internal/gitrepo"
23 "gitlab.com/Timfa/bluegit/internal/images"
24 "gitlab.com/Timfa/bluegit/internal/runnercfg"
25 "gitlab.com/Timfa/bluegit/internal/store"
26 "gitlab.com/Timfa/bluegit/internal/webhook"
27)
28
29// Runner plant jobs en stuurt het resultaat door naar webhooks.
30type Runner struct {
31 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
32 Hold func(store.Pipeline) bool
33 Store *store.Store
34 GitBin string
35 Docker string
36 Storage string
37 BaseURL string
38 Network string
39 Name string
40 Timeout time.Duration
41 Log *slog.Logger
42 OnEvent func(repoID int64, ev webhook.Event)
43 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
44 Presets map[string]images.Override
45 // Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
46 Jobs *runnercfg.Tracker
47 ctl control
48 slots chan struct{}
49 mu sync.RWMutex
50 cancelMu sync.Mutex
51 cancel map[int64]context.CancelFunc
52}
53
54// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
55func (r *Runner) Interrupt(ids []int64) {
56 r.cancelMu.Lock()
57 defer r.cancelMu.Unlock()
58 for _, id := range ids {
59 if c := r.cancel[id]; c != nil {
60 c()
61 }
62 }
63}
64
65func (r *Runner) track(id int64, cancel context.CancelFunc) {
66 r.cancelMu.Lock()
67 defer r.cancelMu.Unlock()
68 if r.cancel == nil {
69 r.cancel = map[int64]context.CancelFunc{}
70 }
71 r.cancel[id] = cancel
72}
73
74func (r *Runner) untrack(id int64) {
75 r.cancelMu.Lock()
76 defer r.cancelMu.Unlock()
77 delete(r.cancel, id)
78}
79
80// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
81func (r *Runner) SetStorage(path string) {
82 r.mu.Lock()
83 r.Storage = path
84 r.mu.Unlock()
85}
86
87func (r *Runner) storage() string {
88 r.mu.RLock()
89 defer r.mu.RUnlock()
90 return r.Storage
91}
92
93// Loop blijft jobs oppakken tot ctx afloopt.
94func (r *Runner) Loop(ctx context.Context, concurrency int) {
95 if concurrency < 1 {
96 concurrency = 1
97 }
98 r.slots = make(chan struct{}, maxSlots)
99 r.mu.Lock()
100 if !r.ctl.set {
101 r.ctl.limit = concurrency
102 }
103 r.mu.Unlock()
104 tick := time.NewTicker(2 * time.Second)
105 defer tick.Stop()
106 for {
107 select {
108 case <-ctx.Done():
109 return
110 case <-tick.C:
111 r.Tick(ctx)
112 }
113 }
114}
115
116// Tick bekijkt alle open pipelines één keer.
117// Zonder opslagpad gebeurt er niets.
118func (r *Runner) Tick(ctx context.Context) {
119 if r.storage() == "" {
120 return
121 }
122 pipes, err := r.Store.ActivePipelines(ctx)
123 if err != nil {
124 r.Log.Error("pipelines", "err", err)
125 return
126 }
127 for _, p := range pipes {
128 r.schedule(ctx, p)
129 }
130 r.reapRemote(ctx)
131}
132
133func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
134 r.settle(ctx, p.ID)
135 if r.Hold != nil && r.Hold(p) {
136 return // wacht op de self-update
137 }
138 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
139 if err != nil {
140 r.Log.Error("jobs", "err", err)
141 return
142 }
143 specs := make([]ci.Job, len(jobs))
144 status := map[string]string{}
145 for i, j := range jobs {
146 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
147 status[j.Name] = j.Status
148 }
149 tierCache := map[int64][]Tier{}
150 for _, j := range jobs {
151 if j.Status != "queued" && j.Status != "manual" {
152 continue
153 }
154 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
155 case "skip":
156 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
157 status[j.Name] = "skipped"
158 case "fail":
159 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
160 status[j.Name] = "failed"
161 case "ready":
162 if j.Status != "queued" {
163 continue
164 }
165 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
166 continue
167 }
168 // Meer jobs dan ci.concurrency blijven queued, met een plek
169 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
170 free := r.acquireSlot()
171 prot := r.refProtected(ctx, j.RepoID, p.Ref)
172 if prot {
173 r.noteProtected(ctx, j.RepoID, j)
174 }
175 if tiers := r.tiersFor(ctx, tierCache, j.RepoID, prot); len(tiers) > 1 {
176 jr := r.Store.JobRun(ctx, j.ID)
177 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
178 if !step.Builtin {
179 if free {
180 r.releaseSlot()
181 }
182 if step.Moved {
183 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
184 }
185 continue
186 }
187 }
188 if !free {
189 continue
190 }
191 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
192 if err != nil || !ok {
193 r.releaseSlot()
194 continue
195 }
196 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
197 _ = r.Store.ClearJobOffer(ctx, j.ID)
198 status[j.Name] = "running"
199 job := j
200 pipe := p
201 go r.execute(ctx, pipe, job)
202 }
203 }
204}
205
206// settle slaat jobs over die door een harde mislukking of een
207// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
208// pipelinestatus al op failed was gezet.
209func (r *Runner) settle(ctx context.Context, pipelineID int64) {
210 for {
211 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
212 if err != nil {
213 if r.Log != nil {
214 r.Log.Error("jobs", "err", err)
215 }
216 return
217 }
218 specs := make([]ci.Job, len(jobs))
219 status := map[string]string{}
220 for i, j := range jobs {
221 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
222 status[j.Name] = j.Status
223 }
224 moved := false
225 for _, j := range jobs {
226 if j.Status != "queued" && j.Status != "manual" {
227 continue
228 }
229 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
230 case "skip":
231 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
232 r.Log.Error("skip", "err", err)
233 continue
234 }
235 status[j.Name] = "skipped"
236 moved = true
237 case "fail":
238 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
239 r.Log.Error("fail", "err", err)
240 continue
241 }
242 status[j.Name] = "failed"
243 moved = true
244 }
245 }
246 if !moved {
247 return
248 }
249 }
250}
251
252// BuiltinName is de naam van de ingebouwde runner in de job-UI.
253func (r *Runner) BuiltinName() string {
254 if r.Name != "" {
255 return r.Name
256 }
257 return "BlueGit runner"
258}
259
260func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64, protected bool) []Tier {
261 key := repoID * 2
262 if protected {
263 key++
264 }
265 if t, ok := cache[key]; ok {
266 return t
267 }
268 var tiers []Tier
269 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
270 tiers, _ = Tiers(ctx, r.Store, repo, protected)
271 }
272 cache[key] = tiers
273 return tiers
274}
275
276// refProtected: draait een pipeline op een beschermde branch (en krijgt
277// hij dus beschermde geheimen)?
278func (r *Runner) refProtected(ctx context.Context, repoID int64, ref string) bool {
279 name, _ := ci.SplitRef(ref)
280 patterns, err := r.Store.ProtectedPatterns(ctx, repoID)
281 if err != nil {
282 return true // bij twijfel streng
283 }
284 return access.BranchProtected(patterns, name)
285}
286
287var protectedNoted sync.Map // job id → melding al in het log
288
289// noteProtected zet één keer een melding in het joblog als er opt-in
290// runners zijn die deze job niet mogen draaien.
291func (r *Runner) noteProtected(ctx context.Context, repoID int64, job store.Job) {
292 if _, done := protectedNoted.LoadOrStore(job.ID, true); done {
293 return
294 }
295 repo, err := r.Store.RepoByID(ctx, repoID)
296 if err != nil {
297 return
298 }
299 if names := ExcludedForProtected(ctx, r.Store, repo); len(names) > 0 {
300 _ = r.Store.AppendJobLog(ctx, job.ID, "Protected branch: this job only runs on the built-in runner, global runners, group runners and account runners of maintainers. Skipped: "+strings.Join(names, ", ")+"\n")
301 }
302}
303
304func indexOf(jobs []store.Job, name string) int {
305 for i, j := range jobs {
306 if j.Name == name {
307 return i
308 }
309 }
310 return 0
311}
312
313func (r *Runner) acquireSlot() bool {
314 if r.Paused() {
315 return false
316 }
317 if r.slots == nil {
318 return true
319 }
320 if l := r.controls().limit; l > 0 && len(r.slots) >= l {
321 return false
322 }
323 select {
324 case r.slots <- struct{}{}:
325 return true
326 default:
327 return false
328 }
329}
330
331func (r *Runner) releaseSlot() {
332 if r.slots == nil {
333 return
334 }
335 select {
336 case <-r.slots:
337 default:
338 }
339}
340
341func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
342 defer r.releaseSlot()
343 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
344 base, stop := context.WithCancel(ctx)
345 r.track(job.ID, stop)
346 defer func() {
347 r.untrack(job.ID)
348 stop()
349 }()
350 runCtx := base
351 limit := r.Timeout
352 if job.Timeout > 0 {
353 limit = job.Timeout
354 }
355 if m := r.controls().timeout; m > 0 && (limit == 0 || time.Duration(m)*time.Minute < limit) {
356 limit = time.Duration(m) * time.Minute
357 }
358 r.trackStart(job)
359 if limit > 0 {
360 var cancel context.CancelFunc
361 runCtx, cancel = context.WithTimeout(base, limit)
362 defer cancel()
363 }
364 mask, err := r.runOne(runCtx, p, job)
365 status := "success"
366 extra := ""
367 kind := ""
368 switch {
369 case err == nil:
370 case ctx.Err() == nil && base.Err() != nil:
371 status = "canceled"
372 extra = stamp("canceled because a newer pipeline started for the same ref")
373 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
374 status = "failed"
375 kind = "job_execution_timeout"
376 extra = stamp(ci.Mask(err.Error(), mask))
377 default:
378 status = "failed"
379 kind = classifyFailure(err)
380 extra = stamp(ci.Mask(err.Error(), mask))
381 }
382 r.trackDone(job.ID, status)
383 r.complete(p, job, before, status, extra, kind)
384}
385
386// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
387// en webhook. Gedeeld door de ingebouwde en de aparte runner.
388func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
389 if status == "failed" && shouldRetry(job, kind) {
390 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
391 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
392 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
393 r.Log.Error("job opnieuw", "err", ferr)
394 } else if ferr == nil {
395 r.settle(context.Background(), p.ID)
396 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
397 r.Log.Error("pipelinestatus", "err", rerr)
398 }
399 return
400 }
401 }
402 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
403 r.Log.Error("job afronden", "err", ferr)
404 }
405 r.settle(context.Background(), p.ID)
406 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
407 if rerr != nil {
408 r.Log.Error("pipelinestatus", "err", rerr)
409 return
410 }
411 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
412 kind := webhook.EventPipelineSuccess
413 word := "geslaagd"
414 if final == "failed" {
415 kind = webhook.EventPipelineFailure
416 word = "mislukt"
417 }
418 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
419 if err != nil {
420 return
421 }
422 branch, _ := gitrepo.BranchOfRef(p.Ref)
423 if branch == "" {
424 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
425 }
426 r.OnEvent(p.RepoID, webhook.Event{
427 Type: kind,
428 Project: repo.FullPath(),
429 Branch: branch,
430 Commit: p.CommitSHA,
431 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
432 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
433 PipelineID: fmt.Sprint(p.ID),
434 })
435 }
436}
437
438func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
439 repo, err := r.Store.RepoByID(ctx, p.RepoID)
440 if err != nil {
441 return nil, err
442 }
443 sink := &logSink{store: r.Store, jobID: job.ID}
444 sink.section("prepare", "Preparing the docker executor")
445 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
446 if err := clearWorkDir(work); err != nil {
447 sink.closeSection()
448 return nil, fmt.Errorf("runner_system_failure: work dir: %w", err)
449 }
450 if err := os.MkdirAll(work, 0o755); err != nil {
451 sink.closeSection()
452 return nil, err
453 }
454 // Na afloop (ook bij falen of annuleren) gaat de werkmap weg. Artifacts
455 // en cache staan dan al gekopieerd buiten de werkmap (collect, saveCache)
456 // en de container is gestopt. Met cleanup_work uit blijft hij staan.
457 defer r.finishWorkDir(work)
458 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
459 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
460 sink.closeSection()
461 return nil, fmt.Errorf("clone: %w", err)
462 }
463 // origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
464 // op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
465 if r.BaseURL != "" {
466 _, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
467 }
468 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
469 sink.closeSection()
470 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
471 }
472 if err := r.restoreCache(repo.ID, job, work); err != nil {
473 sink.closeSection()
474 return nil, err
475 }
476 if err := r.materialize(ctx, p, job, work); err != nil {
477 sink.closeSection()
478 return nil, err
479 }
480 scriptPath := filepath.Join(work, ".bluegit-job.sh")
481 host, _ := os.Hostname()
482 script := shellScript(job, placeNote{
483 Runner: r.Name, Host: host, WorkDir: "/workspace",
484 PipelineID: p.ID, JobID: job.ID,
485 })
486 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
487 sink.closeSection()
488 return nil, err
489 }
490 token, err := r.Store.CreateJobToken(ctx, job.ID)
491 if err != nil {
492 sink.closeSection()
493 return nil, err
494 }
495 env, mask, err := r.environment(ctx, repo, p, job, token)
496 if err != nil {
497 sink.closeSection()
498 return nil, err
499 }
500 sink.secrets = mask
501 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
502 return mask, err
503 }
504 if err := r.collect(ctx, job, work); err != nil {
505 return mask, err
506 }
507 if err := r.saveCache(repo.ID, job, work); err != nil {
508 return mask, err
509 }
510 return mask, nil
511}
512
513// execContainer haalt het image op, start services en draait het
514// jobscript in de container. De sectie prepare moet open staan.
515func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
516 ref, lines, rerr := r.resolveImage(ctx, job.Image)
517 for _, line := range lines {
518 sink.emit(line)
519 }
520 if rerr != nil {
521 sink.closeSection()
522 return rerr
523 }
524 job.Image = ref
525 job.Services = append([]store.Service(nil), job.Services...)
526 for i := range job.Services {
527 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
528 for _, line := range slines {
529 sink.emit(line)
530 }
531 if serr != nil {
532 sink.closeSection()
533 return serr
534 }
535 job.Services[i].Name = sref
536 }
537 network := r.Network
538 if network == "" {
539 network = "bridge"
540 }
541 if len(job.Services) > 0 {
542 net, cleanup, serr := r.startServices(ctx, job)
543 if serr != nil {
544 sink.closeSection()
545 return serr
546 }
547 defer cleanup()
548 network = net
549 }
550 sink.closeSection()
551 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
552 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
553 // staat de waarde niet in de proceslijst.
554 // Een vaste naam per job: een geannuleerde of afgebroken container
555 // (de docker-client stopt dan, de container niet) wordt voor de start en
556 // na afloop opgeruimd.
557 cname := fmt.Sprintf("bluegit-job-%d", job.ID)
558 removeContainer(r.Docker, cname)
559 defer removeContainer(r.Docker, cname)
560 args := []string{"run", "--rm", "--name", cname, "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
561 ctl := r.controls()
562 args = append(args, ctl.runArgs...)
563 if ctl.pull == runnercfg.PullNever {
564 args = append(args, "--pull", "never")
565 }
566 cmdEnv := append([]string{}, os.Environ()...)
567 for k, v := range env {
568 args = append(args, "-e", k)
569 cmdEnv = append(cmdEnv, k+"="+v)
570 }
571 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
572 cmd := exec.CommandContext(ctx, r.Docker, args...)
573 cmd.Env = cmdEnv
574 cmd.Stdout = sink
575 cmd.Stderr = sink
576 err := cmd.Run()
577 sink.flush()
578 if err != nil {
579 return fmt.Errorf("container ended with an error: %w", err)
580 }
581 return nil
582}
583
584func removeContainer(docker, name string) {
585 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
586 defer cancel()
587 _, _ = execOut(ctx, docker, "rm", "-f", name)
588}
589
590// finishWorkDir ruimt de werkmap van een afgelopen job op, plus de lege
591// pipelinemap erboven, tenzij de instelling cleanup_work uit staat.
592func (r *Runner) finishWorkDir(work string) {
593 if r.controls().keep {
594 return
595 }
596 _ = clearWorkDir(work)
597 _ = os.Remove(filepath.Dir(work)) // alleen als hij leeg is
598}
599
600// ClearWorkDir is clearWorkDir voor de aparte runner.
601func ClearWorkDir(work string) error { return clearWorkDir(work) }
602
603// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
604// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
605// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
606// een lege map begint. Alleen als ook dat faalt, is het een fout.
607func clearWorkDir(work string) error {
608 if err := os.RemoveAll(work); err == nil {
609 return nil
610 }
611 _ = filepath.WalkDir(work, func(p string, d fs.DirEntry, err error) error {
612 if err == nil && d.IsDir() {
613 _ = os.Chmod(p, 0o755)
614 }
615 return nil
616 })
617 if err := os.RemoveAll(work); err == nil {
618 return nil
619 }
620 stale := fmt.Sprintf("%s.stale-%d", work, time.Now().UnixNano())
621 if err := os.Rename(work, stale); err != nil {
622 return err
623 }
624 go func() { _ = os.RemoveAll(stale) }()
625 return nil
626}
627
628// placeNote zijn de vaste regels in de sectie Preparing environment.
629type placeNote struct {
630 Runner string
631 Host string
632 WorkDir string
633 PipelineID int64
634 JobID int64
635}
636
637func (n placeNote) lines() []string {
638 runner := strings.TrimSpace(n.Runner)
639 if runner == "" {
640 runner = "Steve"
641 }
642 host := strings.TrimSpace(n.Host)
643 if host == "" {
644 host = "localhost"
645 }
646 dir := strings.TrimSpace(n.WorkDir)
647 if dir == "" {
648 dir = "/workspace"
649 }
650 return []string{
651 fmt.Sprintf("Running on %s via %s...", runner, host),
652 fmt.Sprintf("Working directory: %s", dir),
653 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
654 }
655}
656
657func shSingle(s string) string {
658 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
659}
660
661func shellScript(job store.Job, note placeNote) string {
662 var b strings.Builder
663 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
664 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
665 for _, line := range note.lines() {
666 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
667 }
668 if strings.TrimSpace(job.Before) != "" {
669 b.WriteString("(\nset -e\n")
670 b.WriteString(job.Before)
671 if !strings.HasSuffix(job.Before, "\n") {
672 b.WriteByte('\n')
673 }
674 b.WriteString(")\nbrc=$?\n")
675 } else {
676 b.WriteString("brc=0\n")
677 }
678 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
679 b.WriteString("echo \"section_start:script:Running script\"\n")
680 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
681 b.WriteString(job.Script)
682 if !strings.HasSuffix(job.Script, "\n") {
683 b.WriteByte('\n')
684 }
685 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
686 b.WriteString("echo \"section_end:script:Running script\"\n")
687 b.WriteString("echo \"section_start:after:After script\"\n")
688 if strings.TrimSpace(job.After) != "" {
689 b.WriteString("(\nset -e\n")
690 b.WriteString(job.After)
691 if !strings.HasSuffix(job.After, "\n") {
692 b.WriteByte('\n')
693 }
694 b.WriteString(")\n")
695 }
696 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
697 return b.String()
698}
699
700// resolveImage kiest het image waarmee de container start.
701// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
702func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
703 image = strings.TrimSpace(image)
704 set, err := images.Effective(r.presetSet())
705 if err != nil {
706 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
707 }
708 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
709 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
710 if p, ok := images.Find(set, image); ok {
711 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
712 }
713 return image, dockerImageLines(image, "", nil), missing
714 }
715 p, ok := images.Find(set, image)
716 if !ok {
717 if r.controls().pull == runnercfg.PullAlways {
718 _, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
719 }
720 id, digests := r.inspectImage(ctx, image)
721 return image, dockerImageLines(image, id, digests), nil
722 }
723 ref, built, err := p.Ensure(ctx, r.Docker)
724 if err != nil {
725 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
726 }
727 var lines []string
728 if built {
729 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
730 }
731 id, digests := r.inspectImage(ctx, ref)
732 var baseDigests []string
733 if ref != p.From {
734 _, baseDigests = r.inspectImage(ctx, p.From)
735 }
736 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
737 return ref, lines, nil
738}
739
740func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
741 var lines []string
742 if preset != "" {
743 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
744 }
745 if ref == "" {
746 return lines
747 }
748 lines = append(lines, dockerImageLines(ref, id, digests)...)
749 if preset != "" && ref != base {
750 digest := matchingDigest(base, baseDigests)
751 if digest == "" && len(baseDigests) > 0 {
752 digest = strings.TrimSpace(baseDigests[0])
753 }
754 if digest != "" {
755 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
756 } else {
757 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
758 }
759 }
760 return lines
761}
762
763func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
764 if strings.TrimSpace(image) == "" || r.Docker == "" {
765 return "", nil
766 }
767 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
768 if err != nil {
769 return "", nil
770 }
771 return parseImageInspect(out)
772}
773
774func parseImageInspect(out string) (string, []string) {
775 lines := strings.Split(out, "\n")
776 id := ""
777 if len(lines) > 0 {
778 id = strings.TrimSpace(lines[0])
779 }
780 var digests []string
781 for _, line := range lines[1:] {
782 line = strings.TrimSpace(line)
783 if line != "" {
784 digests = append(digests, line)
785 }
786 }
787 return id, digests
788}
789
790func dockerImageLines(image, id string, digests []string) []string {
791 image = strings.TrimSpace(image)
792 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
793 id = strings.TrimSpace(id)
794 digest := matchingDigest(image, digests)
795 switch {
796 case id != "" && digest != "":
797 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
798 case digest != "":
799 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
800 case id != "":
801 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
802 }
803 return lines
804}
805
806func matchingDigest(image string, digests []string) string {
807 name := imageName(image)
808 fallback := ""
809 for _, digest := range digests {
810 digest = strings.TrimSpace(digest)
811 at := strings.LastIndex(digest, "@")
812 if digest == "" || at <= 0 {
813 continue
814 }
815 if fallback == "" {
816 fallback = digest
817 }
818 repo := digest[:at]
819 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
820 return digest
821 }
822 }
823 return fallback
824}
825
826func imageName(image string) string {
827 image = strings.TrimSpace(image)
828 if i := strings.LastIndex(image, "@"); i >= 0 {
829 image = image[:i]
830 }
831 slash := strings.LastIndex(image, "/")
832 name := image
833 if slash >= 0 {
834 name = image[slash+1:]
835 }
836 if i := strings.Index(name, ":"); i >= 0 {
837 name = name[:i]
838 }
839 return name
840}
841
842func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
843 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
844 if err != nil {
845 return err
846 }
847 wanted := map[string]bool{}
848 if job.NeedsSet {
849 for _, n := range job.Needs {
850 wanted[n] = true
851 }
852 } else {
853 stageOrder := map[string]int{}
854 n := 0
855 for _, j := range jobs {
856 if _, ok := stageOrder[j.Stage]; !ok {
857 stageOrder[j.Stage] = n
858 n++
859 }
860 }
861 my := stageOrder[job.Stage]
862 for _, j := range jobs {
863 if stageOrder[j.Stage] < my {
864 wanted[j.Name] = true
865 }
866 }
867 }
868 for _, j := range jobs {
869 if !wanted[j.Name] || j.Status != "success" {
870 continue
871 }
872 for _, rel := range j.ArtifactPaths {
873 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
874 if err != nil {
875 continue
876 }
877 dst := filepath.Join(work, filepath.FromSlash(rel))
878 if err := copyFile(src, dst); err != nil {
879 return err
880 }
881 }
882 }
883 return nil
884}
885
886func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
887 var expires *time.Time
888 if job.ArtifactExpire > 0 {
889 t := time.Now().Add(job.ArtifactExpire)
890 expires = &t
891 }
892 for _, rel := range job.ArtifactPaths {
893 src := filepath.Join(work, filepath.FromSlash(rel))
894 info, err := os.Stat(src)
895 if err != nil || info.IsDir() {
896 return fmt.Errorf("artifact %s ontbreekt", rel)
897 }
898 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
899 if err := copyFile(src, dst); err != nil {
900 return err
901 }
902 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
903 return err
904 }
905 }
906 return nil
907}
908
909func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
910 name, _ := ci.SplitRef(p.Ref)
911 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
912 if err != nil {
913 return nil, nil, err
914 }
915 refProtected := access.BranchProtected(patterns, name)
916 env := map[string]string{}
917 var mask []string
918 if repo.OwnerKind == "group" {
919 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
920 if err != nil {
921 return nil, nil, err
922 }
923 for k, v := range groupSecrets {
924 env[k] = v
925 mask = append(mask, v)
926 }
927 }
928 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
929 if err != nil {
930 return nil, nil, err
931 }
932 for k, v := range repoSecrets {
933 env[k] = v
934 mask = append(mask, v)
935 }
936 for k, v := range job.Variables {
937 env[k] = v
938 }
939 base := strings.TrimRight(r.BaseURL, "/")
940 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
941 var info *ci.MRInfo
942 if p.MRID != 0 {
943 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
944 target := ""
945 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
946 target = strings.TrimSpace(string(out))
947 }
948 info = &ci.MRInfo{
949 ID: m.ID, IID: m.Number, Title: m.Title,
950 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
951 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
952 TargetSHA: target,
953 }
954 }
955 }
956 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
957 pre := ci.Variables(ci.Ref{
958 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
959 DefaultBranch: repo.DefaultBranch, Source: p.Source,
960 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
961 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
962 })
963 for k, v := range pre {
964 env[k] = v
965 }
966 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
967 env["CI_JOB_NAME"] = job.Name
968 env["CI_JOB_STAGE"] = job.Stage
969 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
970 env["CI_JOB_TOKEN"] = token
971 if job.Environment != "" {
972 env["CI_ENVIRONMENT_NAME"] = job.Environment
973 if job.EnvironmentURL != "" {
974 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
975 }
976 }
977 for k, v := range job.Variables {
978 if strings.HasPrefix(k, "CI_NODE_") {
979 env[k] = v
980 }
981 }
982 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
983 // en safe.directory er alsnog bij komt.
984 markWorkspaceSafe(env)
985 JobGitAuth(env, base+"/"+repo.FullPath(), token)
986 mask = append(mask, token)
987 return env, mask, nil
988}
989
990// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
991// job-token. De header geldt alleen voor de URL van dit project (met en
992// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
993// het in elk image, zonder bestanden in de container.
994func JobGitAuth(env map[string]string, projectURL, token string) {
995 if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
996 return
997 }
998 n := 0
999 if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
1000 n = v
1001 }
1002 for _, u := range []string{projectURL, projectURL + ".git"} {
1003 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
1004 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
1005 n++
1006 }
1007 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
1008}
1009
1010// markWorkspaceSafe zet safe.directory voor elke job.
1011// De checkout is van de host-uid; de container-user is dat niet.
1012// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
1013// * of /workspace zet, blijft zoals hij is.
1014func markWorkspaceSafe(env map[string]string) {
1015 const key = "safe.directory"
1016 n := 0
1017 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
1018 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
1019 n = v
1020 }
1021 }
1022 for i := 0; i < n; i++ {
1023 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
1024 continue
1025 }
1026 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
1027 case "*", "/workspace":
1028 return
1029 }
1030 }
1031 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
1032 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
1033 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
1034}
1035
1036func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
1037 netName := fmt.Sprintf("bluegit-%d", job.ID)
1038 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
1039 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
1040 }
1041 var names []string
1042 cleanup := func() {
1043 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
1044 defer cancel()
1045 for _, name := range names {
1046 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
1047 }
1048 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
1049 }
1050 for i, svc := range job.Services {
1051 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
1052 args := []string{"run", "-d", "--name", cname, "--network", netName}
1053 if svc.Alias != "" {
1054 args = append(args, "--network-alias", svc.Alias)
1055 }
1056 cmdEnv := append([]string{}, os.Environ()...)
1057 for k, v := range svc.Variables {
1058 args = append(args, "-e", k)
1059 cmdEnv = append(cmdEnv, k+"="+v)
1060 }
1061 if len(svc.Entrypoint) > 0 {
1062 args = append(args, "--entrypoint", svc.Entrypoint[0])
1063 }
1064 args = append(args, svc.Name)
1065 if len(svc.Entrypoint) > 1 {
1066 args = append(args, svc.Entrypoint[1:]...)
1067 }
1068 args = append(args, svc.Command...)
1069 cmd := exec.CommandContext(ctx, r.Docker, args...)
1070 cmd.Env = cmdEnv
1071 var stderr bytes.Buffer
1072 cmd.Stderr = &stderr
1073 if err := cmd.Run(); err != nil {
1074 cleanup()
1075 msg := strings.TrimSpace(stderr.String())
1076 if msg == "" {
1077 msg = err.Error()
1078 }
1079 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
1080 }
1081 names = append(names, cname)
1082 }
1083 return netName, cleanup, nil
1084}
1085
1086func shouldRetry(job store.Job, kind string) bool {
1087 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
1088 return false
1089 }
1090 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
1091 return true
1092 }
1093 if containsStr(job.RetryWhen, kind) {
1094 return true
1095 }
1096 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
1097}
1098
1099func classifyFailure(err error) string {
1100 msg := err.Error()
1101 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
1102 return "runner_system_failure"
1103 }
1104 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
1105 return "job_execution_timeout"
1106 }
1107 return "script_failure"
1108}
1109
1110func containsStr(list []string, s string) bool {
1111 for _, item := range list {
1112 if item == s {
1113 return true
1114 }
1115 }
1116 return false
1117}
1118
1119func safeSegment(name string) string {
1120 var b strings.Builder
1121 for _, r := range name {
1122 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1123 b.WriteRune(r)
1124 } else {
1125 b.WriteByte('_')
1126 }
1127 }
1128 if b.Len() == 0 {
1129 return "job"
1130 }
1131 return b.String()
1132}
1133
1134func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
1135 if len(job.CachePaths) == 0 {
1136 return nil
1137 }
1138 root := r.cacheDir(repoID, job.CacheKey)
1139 for _, rel := range job.CachePaths {
1140 src := filepath.Join(root, filepath.FromSlash(rel))
1141 if _, err := os.Stat(src); err != nil {
1142 continue
1143 }
1144 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
1145 return err
1146 }
1147 }
1148 return nil
1149}
1150
1151func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1152 if len(job.CachePaths) == 0 {
1153 return nil
1154 }
1155 root := r.cacheDir(repoID, job.CacheKey)
1156 for _, rel := range job.CachePaths {
1157 src := filepath.Join(work, filepath.FromSlash(rel))
1158 info, err := os.Stat(src)
1159 if err != nil || info.IsDir() {
1160 continue
1161 }
1162 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1163 return err
1164 }
1165 }
1166 return nil
1167}
1168
1169func (r *Runner) cacheDir(repoID int64, key string) string {
1170 if key == "" {
1171 key = "default"
1172 }
1173 var b strings.Builder
1174 for _, r := range key {
1175 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1176 b.WriteRune(r)
1177 }
1178 }
1179 safe := b.String()
1180 if safe == "" {
1181 safe = "default"
1182 }
1183 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1184}
1185
1186func stamp(line string) string {
1187 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1188}
1189
1190type logSink struct {
1191 mu sync.Mutex
1192 store *store.Store
1193 jobID int64
1194 out func(string)
1195 secrets []string
1196 partial string
1197 current string
1198 opened time.Time
1199}
1200
1201func (l *logSink) Write(p []byte) (int, error) {
1202 l.mu.Lock()
1203 defer l.mu.Unlock()
1204 l.partial += string(p)
1205 for {
1206 i := strings.IndexByte(l.partial, '\n')
1207 if i < 0 {
1208 break
1209 }
1210 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1211 l.partial = l.partial[i+1:]
1212 }
1213 return len(p), nil
1214}
1215
1216func (l *logSink) flush() {
1217 l.mu.Lock()
1218 defer l.mu.Unlock()
1219 if strings.TrimSpace(l.partial) != "" {
1220 l.emit(l.partial)
1221 }
1222 l.partial = ""
1223}
1224
1225func (l *logSink) emit(line string) {
1226 line = ci.Mask(line, l.secrets)
1227 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1228 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1229 dur := int(time.Since(l.opened).Seconds())
1230 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1231 l.current = ""
1232 }
1233 if strings.HasPrefix(line, "section_start:") {
1234 l.current = line
1235 l.opened = time.Now()
1236 }
1237 }
1238 if l.out != nil {
1239 l.out(stamp(line))
1240 return
1241 }
1242 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1243}
1244
1245func (l *logSink) section(name, title string) {
1246 l.emit("section_start:" + name + ":" + title)
1247}
1248
1249func (l *logSink) closeSection() {
1250 if l.current == "" {
1251 return
1252 }
1253 name := "prepare"
1254 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1255 name = parts[0]
1256 }
1257 l.emit("section_end:" + name + ":done")
1258}
1259
1260func copyFile(src, dst string) error {
1261 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1262 return err
1263 }
1264 in, err := os.Open(src)
1265 if err != nil {
1266 return err
1267 }
1268 defer in.Close()
1269 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1270 if err != nil {
1271 return err
1272 }
1273 defer out.Close()
1274 _, err = io.Copy(out, in)
1275 return err
1276}
1277
1278func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1279 if bin == "" {
1280 bin = "git"
1281 }
1282 cmd := exec.CommandContext(ctx, bin, args...)
1283 var stdout, stderr bytes.Buffer
1284 cmd.Stdout = &stdout
1285 cmd.Stderr = &stderr
1286 if err := cmd.Run(); err != nil {
1287 msg := strings.TrimSpace(stderr.String())
1288 if msg == "" {
1289 msg = err.Error()
1290 }
1291 return "", fmt.Errorf("%s", msg)
1292 }
1293 return stdout.String(), nil
1294}
1295
1296// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1297// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1298// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1299func cloneArgs(bare, work string) []string {
1300 return []string{"clone", "--no-hardlinks", bare, work}
1301}