BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 36.9 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "os"
13 "os/exec"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "gitlab.com/Timfa/bluegit/internal/access"
21 "gitlab.com/Timfa/bluegit/internal/ci"
22 "gitlab.com/Timfa/bluegit/internal/gitrepo"
23 "gitlab.com/Timfa/bluegit/internal/images"
24 "gitlab.com/Timfa/bluegit/internal/runnercfg"
25 "gitlab.com/Timfa/bluegit/internal/store"
26 "gitlab.com/Timfa/bluegit/internal/webhook"
27)
28
29// Runner plant jobs en stuurt het resultaat door naar webhooks.
30type Runner struct {
31 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
32 Hold func(store.Pipeline) bool
33 Store *store.Store
34 GitBin string
35 Docker string
36 Storage string
37 BaseURL string
38 Network string
39 Name string
40 Timeout time.Duration
41 Log *slog.Logger
42 OnEvent func(repoID int64, ev webhook.Event)
43 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
44 Presets map[string]images.Override
45 // Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
46 Jobs *runnercfg.Tracker
47 ctl control
48 slots chan struct{}
49 mu sync.RWMutex
50 cancelMu sync.Mutex
51 cancel map[int64]context.CancelFunc
52}
53
54// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
55func (r *Runner) Interrupt(ids []int64) {
56 r.cancelMu.Lock()
57 defer r.cancelMu.Unlock()
58 for _, id := range ids {
59 if c := r.cancel[id]; c != nil {
60 c()
61 }
62 }
63}
64
65// cancelReason leest waarom een lopende job gestopt werd. CancelJob (door
66// een gebruiker) zet canceled_by_user al in de database; anders was het
67// een nieuwere pipeline op dezelfde ref.
68func (r *Runner) cancelReason(repoID, id int64) string {
69 if j, err := r.Store.JobByID(context.Background(), repoID, id); err == nil && j.Reason == "canceled_by_user" {
70 return j.Reason
71 }
72 return "canceled_by_newer_pipeline"
73}
74
75func cancelText(reason string) string {
76 if reason == "canceled_by_user" {
77 return "canceled by a user"
78 }
79 return "canceled because a newer pipeline started for the same ref"
80}
81
82func (r *Runner) track(id int64, cancel context.CancelFunc) {
83 r.cancelMu.Lock()
84 defer r.cancelMu.Unlock()
85 if r.cancel == nil {
86 r.cancel = map[int64]context.CancelFunc{}
87 }
88 r.cancel[id] = cancel
89}
90
91func (r *Runner) untrack(id int64) {
92 r.cancelMu.Lock()
93 defer r.cancelMu.Unlock()
94 delete(r.cancel, id)
95}
96
97// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
98func (r *Runner) SetStorage(path string) {
99 r.mu.Lock()
100 r.Storage = path
101 r.mu.Unlock()
102}
103
104func (r *Runner) storage() string {
105 r.mu.RLock()
106 defer r.mu.RUnlock()
107 return r.Storage
108}
109
110// Loop blijft jobs oppakken tot ctx afloopt.
111func (r *Runner) Loop(ctx context.Context, concurrency int) {
112 if concurrency < 1 {
113 concurrency = 1
114 }
115 r.slots = make(chan struct{}, maxSlots)
116 r.mu.Lock()
117 if !r.ctl.set {
118 r.ctl.limit = concurrency
119 }
120 r.mu.Unlock()
121 tick := time.NewTicker(2 * time.Second)
122 defer tick.Stop()
123 for {
124 select {
125 case <-ctx.Done():
126 return
127 case <-tick.C:
128 r.Tick(ctx)
129 }
130 }
131}
132
133// Tick bekijkt alle open pipelines één keer.
134// Zonder opslagpad gebeurt er niets.
135func (r *Runner) Tick(ctx context.Context) {
136 if r.storage() == "" {
137 return
138 }
139 pipes, err := r.Store.ActivePipelines(ctx)
140 if err != nil {
141 r.Log.Error("pipelines", "err", err)
142 return
143 }
144 for _, p := range pipes {
145 r.schedule(ctx, p)
146 }
147 r.reapRemote(ctx)
148}
149
150func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
151 r.settle(ctx, p.ID)
152 if r.Hold != nil && r.Hold(p) {
153 return // wacht op de self-update
154 }
155 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
156 if err != nil {
157 r.Log.Error("jobs", "err", err)
158 return
159 }
160 specs := make([]ci.Job, len(jobs))
161 status := map[string]string{}
162 for i, j := range jobs {
163 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
164 status[j.Name] = j.Status
165 }
166 tierCache := map[int64][]Tier{}
167 for _, j := range jobs {
168 if j.Status != "queued" && j.Status != "manual" {
169 continue
170 }
171 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
172 case "skip":
173 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
174 status[j.Name] = "skipped"
175 case "fail":
176 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
177 status[j.Name] = "failed"
178 case "ready":
179 if j.Status != "queued" {
180 continue
181 }
182 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
183 continue
184 }
185 // Meer jobs dan ci.concurrency blijven queued, met een plek
186 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
187 free := r.acquireSlot()
188 prot := r.refProtected(ctx, j.RepoID, p.Ref)
189 if prot {
190 r.noteProtected(ctx, j.RepoID, j)
191 }
192 if tiers := r.tiersFor(ctx, tierCache, j.RepoID, prot); len(tiers) > 1 {
193 jr := r.Store.JobRun(ctx, j.ID)
194 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
195 if !step.Builtin {
196 if free {
197 r.releaseSlot()
198 }
199 if step.Moved {
200 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
201 }
202 continue
203 }
204 }
205 if !free {
206 continue
207 }
208 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
209 if err != nil || !ok {
210 r.releaseSlot()
211 continue
212 }
213 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
214 _ = r.Store.ClearJobOffer(ctx, j.ID)
215 status[j.Name] = "running"
216 job := j
217 pipe := p
218 go r.execute(ctx, pipe, job)
219 }
220 }
221}
222
223// settle slaat jobs over die door een harde mislukking of een
224// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
225// pipelinestatus al op failed was gezet.
226func (r *Runner) settle(ctx context.Context, pipelineID int64) {
227 for {
228 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
229 if err != nil {
230 if r.Log != nil {
231 r.Log.Error("jobs", "err", err)
232 }
233 return
234 }
235 specs := make([]ci.Job, len(jobs))
236 status := map[string]string{}
237 for i, j := range jobs {
238 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
239 status[j.Name] = j.Status
240 }
241 moved := false
242 for _, j := range jobs {
243 if j.Status != "queued" && j.Status != "manual" {
244 continue
245 }
246 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
247 case "skip":
248 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
249 r.Log.Error("skip", "err", err)
250 continue
251 }
252 status[j.Name] = "skipped"
253 moved = true
254 case "fail":
255 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
256 r.Log.Error("fail", "err", err)
257 continue
258 }
259 status[j.Name] = "failed"
260 moved = true
261 }
262 }
263 if !moved {
264 return
265 }
266 }
267}
268
269// BuiltinName is de naam van de ingebouwde runner in de job-UI.
270func (r *Runner) BuiltinName() string {
271 if r.Name != "" {
272 return r.Name
273 }
274 return "BlueGit runner"
275}
276
277func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64, protected bool) []Tier {
278 key := repoID * 2
279 if protected {
280 key++
281 }
282 if t, ok := cache[key]; ok {
283 return t
284 }
285 var tiers []Tier
286 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
287 tiers, _ = Tiers(ctx, r.Store, repo, protected)
288 }
289 cache[key] = tiers
290 return tiers
291}
292
293// refProtected: draait een pipeline op een beschermde branch (en krijgt
294// hij dus beschermde geheimen)?
295func (r *Runner) refProtected(ctx context.Context, repoID int64, ref string) bool {
296 name, _ := ci.SplitRef(ref)
297 patterns, err := r.Store.ProtectedPatterns(ctx, repoID)
298 if err != nil {
299 return true // bij twijfel streng
300 }
301 return access.BranchProtected(patterns, name)
302}
303
304var protectedNoted sync.Map // job id → melding al in het log
305
306// noteProtected zet één keer een melding in het joblog als er opt-in
307// runners zijn die deze job niet mogen draaien.
308func (r *Runner) noteProtected(ctx context.Context, repoID int64, job store.Job) {
309 if _, done := protectedNoted.LoadOrStore(job.ID, true); done {
310 return
311 }
312 repo, err := r.Store.RepoByID(ctx, repoID)
313 if err != nil {
314 return
315 }
316 if names := ExcludedForProtected(ctx, r.Store, repo); len(names) > 0 {
317 _ = r.Store.AppendJobLog(ctx, job.ID, "Protected branch: this job only runs on the built-in runner, global runners, group runners and account runners of maintainers. Skipped: "+strings.Join(names, ", ")+"\n")
318 }
319}
320
321func indexOf(jobs []store.Job, name string) int {
322 for i, j := range jobs {
323 if j.Name == name {
324 return i
325 }
326 }
327 return 0
328}
329
330func (r *Runner) acquireSlot() bool {
331 if r.Paused() {
332 return false
333 }
334 if r.slots == nil {
335 return true
336 }
337 if l := r.controls().limit; l > 0 && len(r.slots) >= l {
338 return false
339 }
340 select {
341 case r.slots <- struct{}{}:
342 return true
343 default:
344 return false
345 }
346}
347
348func (r *Runner) releaseSlot() {
349 if r.slots == nil {
350 return
351 }
352 select {
353 case <-r.slots:
354 default:
355 }
356}
357
358func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
359 defer r.releaseSlot()
360 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
361 base, stop := context.WithCancel(ctx)
362 r.track(job.ID, stop)
363 defer func() {
364 r.untrack(job.ID)
365 stop()
366 }()
367 runCtx := base
368 limit, source := EffectiveTimeout(job.Timeout, r.Timeout, r.controls().timeout)
369 r.trackStart(job)
370 if limit > 0 {
371 var cancel context.CancelFunc
372 runCtx, cancel = context.WithTimeout(base, limit)
373 defer cancel()
374 }
375 mask, err := r.runOne(runCtx, p, job)
376 status := "success"
377 extra := ""
378 kind := ""
379 switch {
380 case err == nil:
381 case ctx.Err() == nil && base.Err() != nil:
382 status = "canceled"
383 kind = r.cancelReason(job.RepoID, job.ID)
384 extra = stamp(cancelText(kind))
385 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
386 status = "failed"
387 kind = "job_execution_timeout"
388 extra = stamp(TimedOutLine(limit, source))
389 default:
390 status = "failed"
391 kind = classifyFailure(err)
392 extra = stamp(ci.Mask(err.Error(), mask))
393 }
394 r.trackDone(job.ID, status)
395 r.complete(p, job, before, status, extra, kind)
396}
397
398// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
399// en webhook. Gedeeld door de ingebouwde en de aparte runner.
400func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
401 if status == "failed" && shouldRetry(job, kind) {
402 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
403 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
404 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
405 r.Log.Error("job opnieuw", "err", ferr)
406 } else if ferr == nil {
407 r.settle(context.Background(), p.ID)
408 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
409 r.Log.Error("pipelinestatus", "err", rerr)
410 }
411 return
412 }
413 }
414 if ferr := r.Store.FinishJobReason(context.Background(), job.ID, status, extra, kind); ferr != nil {
415 r.Log.Error("job afronden", "err", ferr)
416 }
417 r.settle(context.Background(), p.ID)
418 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
419 if rerr != nil {
420 r.Log.Error("pipelinestatus", "err", rerr)
421 return
422 }
423 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
424 kind := webhook.EventPipelineSuccess
425 word := "geslaagd"
426 if final == "failed" {
427 kind = webhook.EventPipelineFailure
428 word = "mislukt"
429 }
430 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
431 if err != nil {
432 return
433 }
434 branch, _ := gitrepo.BranchOfRef(p.Ref)
435 if branch == "" {
436 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
437 }
438 r.OnEvent(p.RepoID, webhook.Event{
439 Type: kind,
440 Project: repo.FullPath(),
441 Branch: branch,
442 Commit: p.CommitSHA,
443 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
444 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
445 PipelineID: fmt.Sprint(p.ID),
446 })
447 }
448}
449
450func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
451 repo, err := r.Store.RepoByID(ctx, p.RepoID)
452 if err != nil {
453 return nil, err
454 }
455 sink := &logSink{store: r.Store, jobID: job.ID}
456 sink.section("prepare", "Preparing the docker executor")
457 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
458 if err := clearWorkDir(work); err != nil {
459 sink.closeSection()
460 return nil, fmt.Errorf("runner_system_failure: work dir: %w", err)
461 }
462 if err := os.MkdirAll(work, 0o755); err != nil {
463 sink.closeSection()
464 return nil, err
465 }
466 // Na afloop (ook bij falen of annuleren) gaat de werkmap weg. Artifacts
467 // en cache staan dan al gekopieerd buiten de werkmap (collect, saveCache)
468 // en de container is gestopt. Met cleanup_work uit blijft hij staan.
469 defer r.finishWorkDir(work)
470 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
471 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
472 sink.closeSection()
473 return nil, fmt.Errorf("clone: %w", err)
474 }
475 // origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
476 // op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
477 if r.BaseURL != "" {
478 _, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
479 }
480 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
481 sink.closeSection()
482 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
483 }
484 if err := r.restoreCache(repo.ID, job, work); err != nil {
485 sink.closeSection()
486 return nil, err
487 }
488 if err := r.materialize(ctx, p, job, work); err != nil {
489 sink.closeSection()
490 return nil, err
491 }
492 scriptPath := filepath.Join(work, ".bluegit-job.sh")
493 host, _ := os.Hostname()
494 script := shellScript(job, placeNote{
495 Runner: r.Name, Host: host, WorkDir: "/workspace",
496 PipelineID: p.ID, JobID: job.ID,
497 })
498 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
499 sink.closeSection()
500 return nil, err
501 }
502 token, err := r.Store.CreateJobToken(ctx, job.ID)
503 if err != nil {
504 sink.closeSection()
505 return nil, err
506 }
507 env, mask, err := r.environment(ctx, repo, p, job, token)
508 if err != nil {
509 sink.closeSection()
510 return nil, err
511 }
512 sink.secrets = mask
513 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
514 return mask, err
515 }
516 if err := r.collect(ctx, job, work); err != nil {
517 return mask, err
518 }
519 if err := r.saveCache(repo.ID, job, work); err != nil {
520 return mask, err
521 }
522 return mask, nil
523}
524
525// execContainer haalt het image op, start services en draait het
526// jobscript in de container. De sectie prepare moet open staan.
527func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
528 ref, lines, rerr := r.resolveImage(ctx, job.Image)
529 for _, line := range lines {
530 sink.emit(line)
531 }
532 if rerr != nil {
533 sink.closeSection()
534 return rerr
535 }
536 job.Image = ref
537 job.Services = append([]store.Service(nil), job.Services...)
538 for i := range job.Services {
539 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
540 for _, line := range slines {
541 sink.emit(line)
542 }
543 if serr != nil {
544 sink.closeSection()
545 return serr
546 }
547 job.Services[i].Name = sref
548 }
549 network := r.Network
550 if network == "" {
551 network = "bridge"
552 }
553 if len(job.Services) > 0 {
554 net, cleanup, serr := r.startServices(ctx, job)
555 if serr != nil {
556 sink.closeSection()
557 return serr
558 }
559 defer cleanup()
560 network = net
561 }
562 sink.closeSection()
563 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
564 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
565 // staat de waarde niet in de proceslijst.
566 // Een vaste naam per job: een geannuleerde of afgebroken container
567 // (de docker-client stopt dan, de container niet) wordt voor de start en
568 // na afloop opgeruimd.
569 cname := fmt.Sprintf("bluegit-job-%d", job.ID)
570 removeContainer(r.Docker, cname)
571 defer removeContainer(r.Docker, cname)
572 args := []string{"run", "--rm", "--name", cname, "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
573 ctl := r.controls()
574 args = append(args, ctl.runArgs...)
575 if ctl.pull == runnercfg.PullNever {
576 args = append(args, "--pull", "never")
577 }
578 cmdEnv := append([]string{}, os.Environ()...)
579 for k, v := range env {
580 args = append(args, "-e", k)
581 cmdEnv = append(cmdEnv, k+"="+v)
582 }
583 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
584 cmd := exec.CommandContext(ctx, r.Docker, args...)
585 cmd.Env = cmdEnv
586 cmd.Stdout = sink
587 cmd.Stderr = sink
588 err := cmd.Run()
589 sink.flush()
590 if err != nil {
591 return fmt.Errorf("container ended with an error: %w", err)
592 }
593 return nil
594}
595
596func removeContainer(docker, name string) {
597 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
598 defer cancel()
599 _, _ = execOut(ctx, docker, "rm", "-f", name)
600}
601
602// finishWorkDir ruimt de werkmap van een afgelopen job op, plus de lege
603// pipelinemap erboven, tenzij de instelling cleanup_work uit staat.
604func (r *Runner) finishWorkDir(work string) {
605 if r.controls().keep {
606 return
607 }
608 _ = clearWorkDir(work)
609 _ = os.Remove(filepath.Dir(work)) // alleen als hij leeg is
610}
611
612// ClearWorkDir is clearWorkDir voor de aparte runner.
613func ClearWorkDir(work string) error { return clearWorkDir(work) }
614
615// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
616// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
617// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
618// een lege map begint. Alleen als ook dat faalt, is het een fout.
619func clearWorkDir(work string) error {
620 if err := os.RemoveAll(work); err == nil {
621 return nil
622 }
623 _ = filepath.WalkDir(work, func(p string, d fs.DirEntry, err error) error {
624 if err == nil && d.IsDir() {
625 _ = os.Chmod(p, 0o755)
626 }
627 return nil
628 })
629 if err := os.RemoveAll(work); err == nil {
630 return nil
631 }
632 stale := fmt.Sprintf("%s.stale-%d", work, time.Now().UnixNano())
633 if err := os.Rename(work, stale); err != nil {
634 return err
635 }
636 go func() { _ = os.RemoveAll(stale) }()
637 return nil
638}
639
640// placeNote zijn de vaste regels in de sectie Preparing environment.
641type placeNote struct {
642 Runner string
643 Host string
644 WorkDir string
645 PipelineID int64
646 JobID int64
647}
648
649func (n placeNote) lines() []string {
650 runner := strings.TrimSpace(n.Runner)
651 if runner == "" {
652 runner = "Steve"
653 }
654 host := strings.TrimSpace(n.Host)
655 if host == "" {
656 host = "localhost"
657 }
658 dir := strings.TrimSpace(n.WorkDir)
659 if dir == "" {
660 dir = "/workspace"
661 }
662 return []string{
663 fmt.Sprintf("Running on %s via %s...", runner, host),
664 fmt.Sprintf("Working directory: %s", dir),
665 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
666 }
667}
668
669func shSingle(s string) string {
670 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
671}
672
673func shellScript(job store.Job, note placeNote) string {
674 var b strings.Builder
675 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
676 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
677 for _, line := range note.lines() {
678 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
679 }
680 if strings.TrimSpace(job.Before) != "" {
681 b.WriteString("(\nset -e\n")
682 b.WriteString(job.Before)
683 if !strings.HasSuffix(job.Before, "\n") {
684 b.WriteByte('\n')
685 }
686 b.WriteString(")\nbrc=$?\n")
687 } else {
688 b.WriteString("brc=0\n")
689 }
690 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
691 b.WriteString("echo \"section_start:script:Running script\"\n")
692 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
693 b.WriteString(job.Script)
694 if !strings.HasSuffix(job.Script, "\n") {
695 b.WriteByte('\n')
696 }
697 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
698 b.WriteString("echo \"section_end:script:Running script\"\n")
699 b.WriteString("echo \"section_start:after:After script\"\n")
700 if strings.TrimSpace(job.After) != "" {
701 b.WriteString("(\nset -e\n")
702 b.WriteString(job.After)
703 if !strings.HasSuffix(job.After, "\n") {
704 b.WriteByte('\n')
705 }
706 b.WriteString(")\n")
707 }
708 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
709 return b.String()
710}
711
712// resolveImage kiest het image waarmee de container start.
713// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
714func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
715 image = strings.TrimSpace(image)
716 set, err := images.Effective(r.presetSet())
717 if err != nil {
718 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
719 }
720 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
721 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
722 if p, ok := images.Find(set, image); ok {
723 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
724 }
725 return image, dockerImageLines(image, "", nil), missing
726 }
727 p, ok := images.Find(set, image)
728 if !ok {
729 if r.controls().pull == runnercfg.PullAlways {
730 _, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
731 }
732 id, digests := r.inspectImage(ctx, image)
733 return image, dockerImageLines(image, id, digests), nil
734 }
735 ref, built, err := p.Ensure(ctx, r.Docker)
736 if err != nil {
737 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
738 }
739 var lines []string
740 if built {
741 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
742 }
743 id, digests := r.inspectImage(ctx, ref)
744 var baseDigests []string
745 if ref != p.From {
746 _, baseDigests = r.inspectImage(ctx, p.From)
747 }
748 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
749 return ref, lines, nil
750}
751
752func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
753 var lines []string
754 if preset != "" {
755 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
756 }
757 if ref == "" {
758 return lines
759 }
760 lines = append(lines, dockerImageLines(ref, id, digests)...)
761 if preset != "" && ref != base {
762 digest := matchingDigest(base, baseDigests)
763 if digest == "" && len(baseDigests) > 0 {
764 digest = strings.TrimSpace(baseDigests[0])
765 }
766 if digest != "" {
767 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
768 } else {
769 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
770 }
771 }
772 return lines
773}
774
775func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
776 if strings.TrimSpace(image) == "" || r.Docker == "" {
777 return "", nil
778 }
779 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
780 if err != nil {
781 return "", nil
782 }
783 return parseImageInspect(out)
784}
785
786func parseImageInspect(out string) (string, []string) {
787 lines := strings.Split(out, "\n")
788 id := ""
789 if len(lines) > 0 {
790 id = strings.TrimSpace(lines[0])
791 }
792 var digests []string
793 for _, line := range lines[1:] {
794 line = strings.TrimSpace(line)
795 if line != "" {
796 digests = append(digests, line)
797 }
798 }
799 return id, digests
800}
801
802func dockerImageLines(image, id string, digests []string) []string {
803 image = strings.TrimSpace(image)
804 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
805 id = strings.TrimSpace(id)
806 digest := matchingDigest(image, digests)
807 switch {
808 case id != "" && digest != "":
809 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
810 case digest != "":
811 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
812 case id != "":
813 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
814 }
815 return lines
816}
817
818func matchingDigest(image string, digests []string) string {
819 name := imageName(image)
820 fallback := ""
821 for _, digest := range digests {
822 digest = strings.TrimSpace(digest)
823 at := strings.LastIndex(digest, "@")
824 if digest == "" || at <= 0 {
825 continue
826 }
827 if fallback == "" {
828 fallback = digest
829 }
830 repo := digest[:at]
831 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
832 return digest
833 }
834 }
835 return fallback
836}
837
838func imageName(image string) string {
839 image = strings.TrimSpace(image)
840 if i := strings.LastIndex(image, "@"); i >= 0 {
841 image = image[:i]
842 }
843 slash := strings.LastIndex(image, "/")
844 name := image
845 if slash >= 0 {
846 name = image[slash+1:]
847 }
848 if i := strings.Index(name, ":"); i >= 0 {
849 name = name[:i]
850 }
851 return name
852}
853
854func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
855 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
856 if err != nil {
857 return err
858 }
859 wanted := map[string]bool{}
860 if job.NeedsSet {
861 for _, n := range job.Needs {
862 wanted[n] = true
863 }
864 } else {
865 stageOrder := map[string]int{}
866 n := 0
867 for _, j := range jobs {
868 if _, ok := stageOrder[j.Stage]; !ok {
869 stageOrder[j.Stage] = n
870 n++
871 }
872 }
873 my := stageOrder[job.Stage]
874 for _, j := range jobs {
875 if stageOrder[j.Stage] < my {
876 wanted[j.Name] = true
877 }
878 }
879 }
880 for _, j := range jobs {
881 if !wanted[j.Name] || j.Status != "success" {
882 continue
883 }
884 for _, rel := range j.ArtifactPaths {
885 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
886 if err != nil {
887 continue
888 }
889 dst := filepath.Join(work, filepath.FromSlash(rel))
890 if err := copyFile(src, dst); err != nil {
891 return err
892 }
893 }
894 }
895 return nil
896}
897
898func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
899 var expires *time.Time
900 if job.ArtifactExpire > 0 {
901 t := time.Now().Add(job.ArtifactExpire)
902 expires = &t
903 }
904 for _, rel := range job.ArtifactPaths {
905 src := filepath.Join(work, filepath.FromSlash(rel))
906 info, err := os.Stat(src)
907 if err != nil || info.IsDir() {
908 return fmt.Errorf("artifact %s ontbreekt", rel)
909 }
910 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
911 if err := copyFile(src, dst); err != nil {
912 return err
913 }
914 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
915 return err
916 }
917 }
918 return nil
919}
920
921func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
922 name, _ := ci.SplitRef(p.Ref)
923 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
924 if err != nil {
925 return nil, nil, err
926 }
927 refProtected := access.BranchProtected(patterns, name)
928 env := map[string]string{}
929 var mask []string
930 if repo.OwnerKind == "group" {
931 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
932 if err != nil {
933 return nil, nil, err
934 }
935 for k, v := range groupSecrets {
936 env[k] = v
937 mask = append(mask, v)
938 }
939 }
940 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
941 if err != nil {
942 return nil, nil, err
943 }
944 for k, v := range repoSecrets {
945 env[k] = v
946 mask = append(mask, v)
947 }
948 for k, v := range job.Variables {
949 env[k] = v
950 }
951 base := strings.TrimRight(r.BaseURL, "/")
952 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
953 var info *ci.MRInfo
954 if p.MRID != 0 {
955 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
956 target := ""
957 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
958 target = strings.TrimSpace(string(out))
959 }
960 info = &ci.MRInfo{
961 ID: m.ID, IID: m.Number, Title: m.Title,
962 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
963 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
964 TargetSHA: target,
965 }
966 }
967 }
968 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
969 pre := ci.Variables(ci.Ref{
970 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
971 DefaultBranch: repo.DefaultBranch, Source: p.Source,
972 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
973 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
974 })
975 for k, v := range pre {
976 env[k] = v
977 }
978 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
979 env["CI_JOB_NAME"] = job.Name
980 env["CI_JOB_STAGE"] = job.Stage
981 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
982 env["CI_JOB_TOKEN"] = token
983 if job.Environment != "" {
984 env["CI_ENVIRONMENT_NAME"] = job.Environment
985 if job.EnvironmentURL != "" {
986 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
987 }
988 }
989 for k, v := range job.Variables {
990 if strings.HasPrefix(k, "CI_NODE_") {
991 env[k] = v
992 }
993 }
994 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
995 // en safe.directory er alsnog bij komt.
996 markWorkspaceSafe(env)
997 JobGitAuth(env, base+"/"+repo.FullPath(), token)
998 mask = append(mask, token)
999 return env, mask, nil
1000}
1001
1002// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
1003// job-token. De header geldt alleen voor de URL van dit project (met en
1004// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
1005// het in elk image, zonder bestanden in de container.
1006func JobGitAuth(env map[string]string, projectURL, token string) {
1007 if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
1008 return
1009 }
1010 n := 0
1011 if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
1012 n = v
1013 }
1014 for _, u := range []string{projectURL, projectURL + ".git"} {
1015 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
1016 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
1017 n++
1018 }
1019 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
1020}
1021
1022// markWorkspaceSafe zet safe.directory voor elke job.
1023// De checkout is van de host-uid; de container-user is dat niet.
1024// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
1025// * of /workspace zet, blijft zoals hij is.
1026func markWorkspaceSafe(env map[string]string) {
1027 const key = "safe.directory"
1028 n := 0
1029 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
1030 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
1031 n = v
1032 }
1033 }
1034 for i := 0; i < n; i++ {
1035 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
1036 continue
1037 }
1038 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
1039 case "*", "/workspace":
1040 return
1041 }
1042 }
1043 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
1044 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
1045 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
1046}
1047
1048func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
1049 netName := fmt.Sprintf("bluegit-%d", job.ID)
1050 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
1051 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
1052 }
1053 var names []string
1054 cleanup := func() {
1055 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
1056 defer cancel()
1057 for _, name := range names {
1058 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
1059 }
1060 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
1061 }
1062 for i, svc := range job.Services {
1063 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
1064 args := []string{"run", "-d", "--name", cname, "--network", netName}
1065 if svc.Alias != "" {
1066 args = append(args, "--network-alias", svc.Alias)
1067 }
1068 cmdEnv := append([]string{}, os.Environ()...)
1069 for k, v := range svc.Variables {
1070 args = append(args, "-e", k)
1071 cmdEnv = append(cmdEnv, k+"="+v)
1072 }
1073 if len(svc.Entrypoint) > 0 {
1074 args = append(args, "--entrypoint", svc.Entrypoint[0])
1075 }
1076 args = append(args, svc.Name)
1077 if len(svc.Entrypoint) > 1 {
1078 args = append(args, svc.Entrypoint[1:]...)
1079 }
1080 args = append(args, svc.Command...)
1081 cmd := exec.CommandContext(ctx, r.Docker, args...)
1082 cmd.Env = cmdEnv
1083 var stderr bytes.Buffer
1084 cmd.Stderr = &stderr
1085 if err := cmd.Run(); err != nil {
1086 cleanup()
1087 msg := strings.TrimSpace(stderr.String())
1088 if msg == "" {
1089 msg = err.Error()
1090 }
1091 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
1092 }
1093 names = append(names, cname)
1094 }
1095 return netName, cleanup, nil
1096}
1097
1098func shouldRetry(job store.Job, kind string) bool {
1099 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
1100 return false
1101 }
1102 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
1103 return true
1104 }
1105 if containsStr(job.RetryWhen, kind) {
1106 return true
1107 }
1108 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
1109}
1110
1111func classifyFailure(err error) string {
1112 msg := err.Error()
1113 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
1114 return "runner_system_failure"
1115 }
1116 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
1117 return "job_execution_timeout"
1118 }
1119 return "script_failure"
1120}
1121
1122func containsStr(list []string, s string) bool {
1123 for _, item := range list {
1124 if item == s {
1125 return true
1126 }
1127 }
1128 return false
1129}
1130
1131func safeSegment(name string) string {
1132 var b strings.Builder
1133 for _, r := range name {
1134 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1135 b.WriteRune(r)
1136 } else {
1137 b.WriteByte('_')
1138 }
1139 }
1140 if b.Len() == 0 {
1141 return "job"
1142 }
1143 return b.String()
1144}
1145
1146func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
1147 if len(job.CachePaths) == 0 {
1148 return nil
1149 }
1150 root := r.cacheDir(repoID, job.CacheKey)
1151 for _, rel := range job.CachePaths {
1152 src := filepath.Join(root, filepath.FromSlash(rel))
1153 if _, err := os.Stat(src); err != nil {
1154 continue
1155 }
1156 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
1157 return err
1158 }
1159 }
1160 return nil
1161}
1162
1163func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1164 if len(job.CachePaths) == 0 {
1165 return nil
1166 }
1167 root := r.cacheDir(repoID, job.CacheKey)
1168 for _, rel := range job.CachePaths {
1169 src := filepath.Join(work, filepath.FromSlash(rel))
1170 info, err := os.Stat(src)
1171 if err != nil || info.IsDir() {
1172 continue
1173 }
1174 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1175 return err
1176 }
1177 }
1178 return nil
1179}
1180
1181func (r *Runner) cacheDir(repoID int64, key string) string {
1182 if key == "" {
1183 key = "default"
1184 }
1185 var b strings.Builder
1186 for _, r := range key {
1187 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1188 b.WriteRune(r)
1189 }
1190 }
1191 safe := b.String()
1192 if safe == "" {
1193 safe = "default"
1194 }
1195 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1196}
1197
1198func stamp(line string) string {
1199 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1200}
1201
1202type logSink struct {
1203 mu sync.Mutex
1204 store *store.Store
1205 jobID int64
1206 out func(string)
1207 secrets []string
1208 partial string
1209 current string
1210 opened time.Time
1211}
1212
1213func (l *logSink) Write(p []byte) (int, error) {
1214 l.mu.Lock()
1215 defer l.mu.Unlock()
1216 l.partial += string(p)
1217 for {
1218 i := strings.IndexByte(l.partial, '\n')
1219 if i < 0 {
1220 break
1221 }
1222 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1223 l.partial = l.partial[i+1:]
1224 }
1225 return len(p), nil
1226}
1227
1228func (l *logSink) flush() {
1229 l.mu.Lock()
1230 defer l.mu.Unlock()
1231 if strings.TrimSpace(l.partial) != "" {
1232 l.emit(l.partial)
1233 }
1234 l.partial = ""
1235}
1236
1237func (l *logSink) emit(line string) {
1238 line = ci.Mask(line, l.secrets)
1239 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1240 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1241 dur := int(time.Since(l.opened).Seconds())
1242 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1243 l.current = ""
1244 }
1245 if strings.HasPrefix(line, "section_start:") {
1246 l.current = line
1247 l.opened = time.Now()
1248 }
1249 }
1250 if l.out != nil {
1251 l.out(stamp(line))
1252 return
1253 }
1254 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1255}
1256
1257func (l *logSink) section(name, title string) {
1258 l.emit("section_start:" + name + ":" + title)
1259}
1260
1261func (l *logSink) closeSection() {
1262 if l.current == "" {
1263 return
1264 }
1265 name := "prepare"
1266 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1267 name = parts[0]
1268 }
1269 l.emit("section_end:" + name + ":done")
1270}
1271
1272func copyFile(src, dst string) error {
1273 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1274 return err
1275 }
1276 in, err := os.Open(src)
1277 if err != nil {
1278 return err
1279 }
1280 defer in.Close()
1281 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1282 if err != nil {
1283 return err
1284 }
1285 defer out.Close()
1286 _, err = io.Copy(out, in)
1287 return err
1288}
1289
1290func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1291 if bin == "" {
1292 bin = "git"
1293 }
1294 cmd := exec.CommandContext(ctx, bin, args...)
1295 var stdout, stderr bytes.Buffer
1296 cmd.Stdout = &stdout
1297 cmd.Stderr = &stderr
1298 if err := cmd.Run(); err != nil {
1299 msg := strings.TrimSpace(stderr.String())
1300 if msg == "" {
1301 msg = err.Error()
1302 }
1303 return "", fmt.Errorf("%s", msg)
1304 }
1305 return stdout.String(), nil
1306}
1307
1308// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1309// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1310// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1311func cloneArgs(bare, work string) []string {
1312 return []string{"clone", "--no-hardlinks", bare, work}
1313}