BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 34.5 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "os"
13 "os/exec"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "gitlab.com/Timfa/bluegit/internal/access"
21 "gitlab.com/Timfa/bluegit/internal/ci"
22 "gitlab.com/Timfa/bluegit/internal/gitrepo"
23 "gitlab.com/Timfa/bluegit/internal/images"
24 "gitlab.com/Timfa/bluegit/internal/runnercfg"
25 "gitlab.com/Timfa/bluegit/internal/store"
26 "gitlab.com/Timfa/bluegit/internal/webhook"
27)
28
29// Runner plant jobs en stuurt het resultaat door naar webhooks.
30type Runner struct {
31 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
32 Hold func(store.Pipeline) bool
33 Store *store.Store
34 GitBin string
35 Docker string
36 Storage string
37 BaseURL string
38 Network string
39 Name string
40 Timeout time.Duration
41 Log *slog.Logger
42 OnEvent func(repoID int64, ev webhook.Event)
43 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
44 Presets map[string]images.Override
45 // Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
46 Jobs *runnercfg.Tracker
47 ctl control
48 slots chan struct{}
49 mu sync.RWMutex
50 cancelMu sync.Mutex
51 cancel map[int64]context.CancelFunc
52}
53
54// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
55func (r *Runner) Interrupt(ids []int64) {
56 r.cancelMu.Lock()
57 defer r.cancelMu.Unlock()
58 for _, id := range ids {
59 if c := r.cancel[id]; c != nil {
60 c()
61 }
62 }
63}
64
65func (r *Runner) track(id int64, cancel context.CancelFunc) {
66 r.cancelMu.Lock()
67 defer r.cancelMu.Unlock()
68 if r.cancel == nil {
69 r.cancel = map[int64]context.CancelFunc{}
70 }
71 r.cancel[id] = cancel
72}
73
74func (r *Runner) untrack(id int64) {
75 r.cancelMu.Lock()
76 defer r.cancelMu.Unlock()
77 delete(r.cancel, id)
78}
79
80// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
81func (r *Runner) SetStorage(path string) {
82 r.mu.Lock()
83 r.Storage = path
84 r.mu.Unlock()
85}
86
87func (r *Runner) storage() string {
88 r.mu.RLock()
89 defer r.mu.RUnlock()
90 return r.Storage
91}
92
93// Loop blijft jobs oppakken tot ctx afloopt.
94func (r *Runner) Loop(ctx context.Context, concurrency int) {
95 if concurrency < 1 {
96 concurrency = 1
97 }
98 r.slots = make(chan struct{}, maxSlots)
99 r.mu.Lock()
100 if !r.ctl.set {
101 r.ctl.limit = concurrency
102 }
103 r.mu.Unlock()
104 tick := time.NewTicker(2 * time.Second)
105 defer tick.Stop()
106 for {
107 select {
108 case <-ctx.Done():
109 return
110 case <-tick.C:
111 r.Tick(ctx)
112 }
113 }
114}
115
116// Tick bekijkt alle open pipelines één keer.
117// Zonder opslagpad gebeurt er niets.
118func (r *Runner) Tick(ctx context.Context) {
119 if r.storage() == "" {
120 return
121 }
122 pipes, err := r.Store.ActivePipelines(ctx)
123 if err != nil {
124 r.Log.Error("pipelines", "err", err)
125 return
126 }
127 for _, p := range pipes {
128 r.schedule(ctx, p)
129 }
130 r.reapRemote(ctx)
131}
132
133func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
134 r.settle(ctx, p.ID)
135 if r.Hold != nil && r.Hold(p) {
136 return // wacht op de self-update
137 }
138 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
139 if err != nil {
140 r.Log.Error("jobs", "err", err)
141 return
142 }
143 specs := make([]ci.Job, len(jobs))
144 status := map[string]string{}
145 for i, j := range jobs {
146 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
147 status[j.Name] = j.Status
148 }
149 tierCache := map[int64][]Tier{}
150 for _, j := range jobs {
151 if j.Status != "queued" && j.Status != "manual" {
152 continue
153 }
154 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
155 case "skip":
156 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
157 status[j.Name] = "skipped"
158 case "fail":
159 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
160 status[j.Name] = "failed"
161 case "ready":
162 if j.Status != "queued" {
163 continue
164 }
165 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
166 continue
167 }
168 // Meer jobs dan ci.concurrency blijven queued, met een plek
169 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
170 free := r.acquireSlot()
171 if tiers := r.tiersFor(ctx, tierCache, j.RepoID); len(tiers) > 1 {
172 jr := r.Store.JobRun(ctx, j.ID)
173 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
174 if !step.Builtin {
175 if free {
176 r.releaseSlot()
177 }
178 if step.Moved {
179 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
180 }
181 continue
182 }
183 }
184 if !free {
185 continue
186 }
187 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
188 if err != nil || !ok {
189 r.releaseSlot()
190 continue
191 }
192 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
193 _ = r.Store.ClearJobOffer(ctx, j.ID)
194 status[j.Name] = "running"
195 job := j
196 pipe := p
197 go r.execute(ctx, pipe, job)
198 }
199 }
200}
201
202// settle slaat jobs over die door een harde mislukking of een
203// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
204// pipelinestatus al op failed was gezet.
205func (r *Runner) settle(ctx context.Context, pipelineID int64) {
206 for {
207 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
208 if err != nil {
209 if r.Log != nil {
210 r.Log.Error("jobs", "err", err)
211 }
212 return
213 }
214 specs := make([]ci.Job, len(jobs))
215 status := map[string]string{}
216 for i, j := range jobs {
217 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
218 status[j.Name] = j.Status
219 }
220 moved := false
221 for _, j := range jobs {
222 if j.Status != "queued" && j.Status != "manual" {
223 continue
224 }
225 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
226 case "skip":
227 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
228 r.Log.Error("skip", "err", err)
229 continue
230 }
231 status[j.Name] = "skipped"
232 moved = true
233 case "fail":
234 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
235 r.Log.Error("fail", "err", err)
236 continue
237 }
238 status[j.Name] = "failed"
239 moved = true
240 }
241 }
242 if !moved {
243 return
244 }
245 }
246}
247
248// BuiltinName is de naam van de ingebouwde runner in de job-UI.
249func (r *Runner) BuiltinName() string {
250 if r.Name != "" {
251 return r.Name
252 }
253 return "BlueGit runner"
254}
255
256func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64) []Tier {
257 if t, ok := cache[repoID]; ok {
258 return t
259 }
260 var tiers []Tier
261 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
262 tiers, _ = Tiers(ctx, r.Store, repo)
263 }
264 cache[repoID] = tiers
265 return tiers
266}
267
268func indexOf(jobs []store.Job, name string) int {
269 for i, j := range jobs {
270 if j.Name == name {
271 return i
272 }
273 }
274 return 0
275}
276
277func (r *Runner) acquireSlot() bool {
278 if r.Paused() {
279 return false
280 }
281 if r.slots == nil {
282 return true
283 }
284 if l := r.controls().limit; l > 0 && len(r.slots) >= l {
285 return false
286 }
287 select {
288 case r.slots <- struct{}{}:
289 return true
290 default:
291 return false
292 }
293}
294
295func (r *Runner) releaseSlot() {
296 if r.slots == nil {
297 return
298 }
299 select {
300 case <-r.slots:
301 default:
302 }
303}
304
305func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
306 defer r.releaseSlot()
307 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
308 base, stop := context.WithCancel(ctx)
309 r.track(job.ID, stop)
310 defer func() {
311 r.untrack(job.ID)
312 stop()
313 }()
314 runCtx := base
315 limit := r.Timeout
316 if job.Timeout > 0 {
317 limit = job.Timeout
318 }
319 if m := r.controls().timeout; m > 0 && (limit == 0 || time.Duration(m)*time.Minute < limit) {
320 limit = time.Duration(m) * time.Minute
321 }
322 r.trackStart(job)
323 if limit > 0 {
324 var cancel context.CancelFunc
325 runCtx, cancel = context.WithTimeout(base, limit)
326 defer cancel()
327 }
328 mask, err := r.runOne(runCtx, p, job)
329 status := "success"
330 extra := ""
331 kind := ""
332 switch {
333 case err == nil:
334 case ctx.Err() == nil && base.Err() != nil:
335 status = "canceled"
336 extra = stamp("canceled because a newer pipeline started for the same ref")
337 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
338 status = "failed"
339 kind = "job_execution_timeout"
340 extra = stamp(ci.Mask(err.Error(), mask))
341 default:
342 status = "failed"
343 kind = classifyFailure(err)
344 extra = stamp(ci.Mask(err.Error(), mask))
345 }
346 r.trackDone(job.ID, status)
347 r.complete(p, job, before, status, extra, kind)
348}
349
350// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
351// en webhook. Gedeeld door de ingebouwde en de aparte runner.
352func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
353 if status == "failed" && shouldRetry(job, kind) {
354 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
355 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
356 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
357 r.Log.Error("job opnieuw", "err", ferr)
358 } else if ferr == nil {
359 r.settle(context.Background(), p.ID)
360 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
361 r.Log.Error("pipelinestatus", "err", rerr)
362 }
363 return
364 }
365 }
366 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
367 r.Log.Error("job afronden", "err", ferr)
368 }
369 r.settle(context.Background(), p.ID)
370 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
371 if rerr != nil {
372 r.Log.Error("pipelinestatus", "err", rerr)
373 return
374 }
375 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
376 kind := webhook.EventPipelineSuccess
377 word := "geslaagd"
378 if final == "failed" {
379 kind = webhook.EventPipelineFailure
380 word = "mislukt"
381 }
382 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
383 if err != nil {
384 return
385 }
386 branch, _ := gitrepo.BranchOfRef(p.Ref)
387 if branch == "" {
388 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
389 }
390 r.OnEvent(p.RepoID, webhook.Event{
391 Type: kind,
392 Project: repo.FullPath(),
393 Branch: branch,
394 Commit: p.CommitSHA,
395 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
396 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
397 PipelineID: fmt.Sprint(p.ID),
398 })
399 }
400}
401
402func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
403 repo, err := r.Store.RepoByID(ctx, p.RepoID)
404 if err != nil {
405 return nil, err
406 }
407 sink := &logSink{store: r.Store, jobID: job.ID}
408 sink.section("prepare", "Preparing the docker executor")
409 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
410 if err := clearWorkDir(work); err != nil {
411 sink.closeSection()
412 return nil, fmt.Errorf("runner_system_failure: work dir: %w", err)
413 }
414 if err := os.MkdirAll(work, 0o755); err != nil {
415 sink.closeSection()
416 return nil, err
417 }
418 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
419 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
420 sink.closeSection()
421 return nil, fmt.Errorf("clone: %w", err)
422 }
423 // origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
424 // op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
425 if r.BaseURL != "" {
426 _, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
427 }
428 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
429 sink.closeSection()
430 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
431 }
432 if err := r.restoreCache(repo.ID, job, work); err != nil {
433 sink.closeSection()
434 return nil, err
435 }
436 if err := r.materialize(ctx, p, job, work); err != nil {
437 sink.closeSection()
438 return nil, err
439 }
440 scriptPath := filepath.Join(work, ".bluegit-job.sh")
441 host, _ := os.Hostname()
442 script := shellScript(job, placeNote{
443 Runner: r.Name, Host: host, WorkDir: "/workspace",
444 PipelineID: p.ID, JobID: job.ID,
445 })
446 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
447 sink.closeSection()
448 return nil, err
449 }
450 token, err := r.Store.CreateJobToken(ctx, job.ID)
451 if err != nil {
452 sink.closeSection()
453 return nil, err
454 }
455 env, mask, err := r.environment(ctx, repo, p, job, token)
456 if err != nil {
457 sink.closeSection()
458 return nil, err
459 }
460 sink.secrets = mask
461 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
462 return mask, err
463 }
464 if err := r.collect(ctx, job, work); err != nil {
465 return mask, err
466 }
467 if err := r.saveCache(repo.ID, job, work); err != nil {
468 return mask, err
469 }
470 return mask, nil
471}
472
473// execContainer haalt het image op, start services en draait het
474// jobscript in de container. De sectie prepare moet open staan.
475func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
476 ref, lines, rerr := r.resolveImage(ctx, job.Image)
477 for _, line := range lines {
478 sink.emit(line)
479 }
480 if rerr != nil {
481 sink.closeSection()
482 return rerr
483 }
484 job.Image = ref
485 job.Services = append([]store.Service(nil), job.Services...)
486 for i := range job.Services {
487 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
488 for _, line := range slines {
489 sink.emit(line)
490 }
491 if serr != nil {
492 sink.closeSection()
493 return serr
494 }
495 job.Services[i].Name = sref
496 }
497 network := r.Network
498 if network == "" {
499 network = "bridge"
500 }
501 if len(job.Services) > 0 {
502 net, cleanup, serr := r.startServices(ctx, job)
503 if serr != nil {
504 sink.closeSection()
505 return serr
506 }
507 defer cleanup()
508 network = net
509 }
510 sink.closeSection()
511 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
512 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
513 // staat de waarde niet in de proceslijst.
514 // Een vaste naam per job: een geannuleerde of afgebroken container
515 // (de docker-client stopt dan, de container niet) wordt voor de start en
516 // na afloop opgeruimd.
517 cname := fmt.Sprintf("bluegit-job-%d", job.ID)
518 removeContainer(r.Docker, cname)
519 defer removeContainer(r.Docker, cname)
520 args := []string{"run", "--rm", "--name", cname, "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
521 ctl := r.controls()
522 args = append(args, ctl.runArgs...)
523 if ctl.pull == runnercfg.PullNever {
524 args = append(args, "--pull", "never")
525 }
526 cmdEnv := append([]string{}, os.Environ()...)
527 for k, v := range env {
528 args = append(args, "-e", k)
529 cmdEnv = append(cmdEnv, k+"="+v)
530 }
531 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
532 cmd := exec.CommandContext(ctx, r.Docker, args...)
533 cmd.Env = cmdEnv
534 cmd.Stdout = sink
535 cmd.Stderr = sink
536 err := cmd.Run()
537 sink.flush()
538 if err != nil {
539 return fmt.Errorf("container ended with an error: %w", err)
540 }
541 return nil
542}
543
544func removeContainer(docker, name string) {
545 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
546 defer cancel()
547 _, _ = execOut(ctx, docker, "rm", "-f", name)
548}
549
550// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
551// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
552// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
553// een lege map begint. Alleen als ook dat faalt, is het een fout.
554func clearWorkDir(work string) error {
555 if err := os.RemoveAll(work); err == nil {
556 return nil
557 }
558 _ = filepath.WalkDir(work, func(p string, d fs.DirEntry, err error) error {
559 if err == nil && d.IsDir() {
560 _ = os.Chmod(p, 0o755)
561 }
562 return nil
563 })
564 if err := os.RemoveAll(work); err == nil {
565 return nil
566 }
567 stale := fmt.Sprintf("%s.stale-%d", work, time.Now().UnixNano())
568 if err := os.Rename(work, stale); err != nil {
569 return err
570 }
571 go func() { _ = os.RemoveAll(stale) }()
572 return nil
573}
574
575// placeNote zijn de vaste regels in de sectie Preparing environment.
576type placeNote struct {
577 Runner string
578 Host string
579 WorkDir string
580 PipelineID int64
581 JobID int64
582}
583
584func (n placeNote) lines() []string {
585 runner := strings.TrimSpace(n.Runner)
586 if runner == "" {
587 runner = "Steve"
588 }
589 host := strings.TrimSpace(n.Host)
590 if host == "" {
591 host = "localhost"
592 }
593 dir := strings.TrimSpace(n.WorkDir)
594 if dir == "" {
595 dir = "/workspace"
596 }
597 return []string{
598 fmt.Sprintf("Running on %s via %s...", runner, host),
599 fmt.Sprintf("Working directory: %s", dir),
600 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
601 }
602}
603
604func shSingle(s string) string {
605 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
606}
607
608func shellScript(job store.Job, note placeNote) string {
609 var b strings.Builder
610 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
611 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
612 for _, line := range note.lines() {
613 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
614 }
615 if strings.TrimSpace(job.Before) != "" {
616 b.WriteString("(\nset -e\n")
617 b.WriteString(job.Before)
618 if !strings.HasSuffix(job.Before, "\n") {
619 b.WriteByte('\n')
620 }
621 b.WriteString(")\nbrc=$?\n")
622 } else {
623 b.WriteString("brc=0\n")
624 }
625 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
626 b.WriteString("echo \"section_start:script:Running script\"\n")
627 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
628 b.WriteString(job.Script)
629 if !strings.HasSuffix(job.Script, "\n") {
630 b.WriteByte('\n')
631 }
632 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
633 b.WriteString("echo \"section_end:script:Running script\"\n")
634 b.WriteString("echo \"section_start:after:After script\"\n")
635 if strings.TrimSpace(job.After) != "" {
636 b.WriteString("(\nset -e\n")
637 b.WriteString(job.After)
638 if !strings.HasSuffix(job.After, "\n") {
639 b.WriteByte('\n')
640 }
641 b.WriteString(")\n")
642 }
643 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
644 return b.String()
645}
646
647// resolveImage kiest het image waarmee de container start.
648// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
649func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
650 image = strings.TrimSpace(image)
651 set, err := images.Effective(r.presetSet())
652 if err != nil {
653 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
654 }
655 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
656 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
657 if p, ok := images.Find(set, image); ok {
658 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
659 }
660 return image, dockerImageLines(image, "", nil), missing
661 }
662 p, ok := images.Find(set, image)
663 if !ok {
664 if r.controls().pull == runnercfg.PullAlways {
665 _, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
666 }
667 id, digests := r.inspectImage(ctx, image)
668 return image, dockerImageLines(image, id, digests), nil
669 }
670 ref, built, err := p.Ensure(ctx, r.Docker)
671 if err != nil {
672 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
673 }
674 var lines []string
675 if built {
676 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
677 }
678 id, digests := r.inspectImage(ctx, ref)
679 var baseDigests []string
680 if ref != p.From {
681 _, baseDigests = r.inspectImage(ctx, p.From)
682 }
683 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
684 return ref, lines, nil
685}
686
687func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
688 var lines []string
689 if preset != "" {
690 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
691 }
692 if ref == "" {
693 return lines
694 }
695 lines = append(lines, dockerImageLines(ref, id, digests)...)
696 if preset != "" && ref != base {
697 digest := matchingDigest(base, baseDigests)
698 if digest == "" && len(baseDigests) > 0 {
699 digest = strings.TrimSpace(baseDigests[0])
700 }
701 if digest != "" {
702 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
703 } else {
704 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
705 }
706 }
707 return lines
708}
709
710func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
711 if strings.TrimSpace(image) == "" || r.Docker == "" {
712 return "", nil
713 }
714 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
715 if err != nil {
716 return "", nil
717 }
718 return parseImageInspect(out)
719}
720
721func parseImageInspect(out string) (string, []string) {
722 lines := strings.Split(out, "\n")
723 id := ""
724 if len(lines) > 0 {
725 id = strings.TrimSpace(lines[0])
726 }
727 var digests []string
728 for _, line := range lines[1:] {
729 line = strings.TrimSpace(line)
730 if line != "" {
731 digests = append(digests, line)
732 }
733 }
734 return id, digests
735}
736
737func dockerImageLines(image, id string, digests []string) []string {
738 image = strings.TrimSpace(image)
739 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
740 id = strings.TrimSpace(id)
741 digest := matchingDigest(image, digests)
742 switch {
743 case id != "" && digest != "":
744 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
745 case digest != "":
746 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
747 case id != "":
748 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
749 }
750 return lines
751}
752
753func matchingDigest(image string, digests []string) string {
754 name := imageName(image)
755 fallback := ""
756 for _, digest := range digests {
757 digest = strings.TrimSpace(digest)
758 at := strings.LastIndex(digest, "@")
759 if digest == "" || at <= 0 {
760 continue
761 }
762 if fallback == "" {
763 fallback = digest
764 }
765 repo := digest[:at]
766 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
767 return digest
768 }
769 }
770 return fallback
771}
772
773func imageName(image string) string {
774 image = strings.TrimSpace(image)
775 if i := strings.LastIndex(image, "@"); i >= 0 {
776 image = image[:i]
777 }
778 slash := strings.LastIndex(image, "/")
779 name := image
780 if slash >= 0 {
781 name = image[slash+1:]
782 }
783 if i := strings.Index(name, ":"); i >= 0 {
784 name = name[:i]
785 }
786 return name
787}
788
789func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
790 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
791 if err != nil {
792 return err
793 }
794 wanted := map[string]bool{}
795 if job.NeedsSet {
796 for _, n := range job.Needs {
797 wanted[n] = true
798 }
799 } else {
800 stageOrder := map[string]int{}
801 n := 0
802 for _, j := range jobs {
803 if _, ok := stageOrder[j.Stage]; !ok {
804 stageOrder[j.Stage] = n
805 n++
806 }
807 }
808 my := stageOrder[job.Stage]
809 for _, j := range jobs {
810 if stageOrder[j.Stage] < my {
811 wanted[j.Name] = true
812 }
813 }
814 }
815 for _, j := range jobs {
816 if !wanted[j.Name] || j.Status != "success" {
817 continue
818 }
819 for _, rel := range j.ArtifactPaths {
820 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
821 if err != nil {
822 continue
823 }
824 dst := filepath.Join(work, filepath.FromSlash(rel))
825 if err := copyFile(src, dst); err != nil {
826 return err
827 }
828 }
829 }
830 return nil
831}
832
833func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
834 var expires *time.Time
835 if job.ArtifactExpire > 0 {
836 t := time.Now().Add(job.ArtifactExpire)
837 expires = &t
838 }
839 for _, rel := range job.ArtifactPaths {
840 src := filepath.Join(work, filepath.FromSlash(rel))
841 info, err := os.Stat(src)
842 if err != nil || info.IsDir() {
843 return fmt.Errorf("artifact %s ontbreekt", rel)
844 }
845 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
846 if err := copyFile(src, dst); err != nil {
847 return err
848 }
849 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
850 return err
851 }
852 }
853 return nil
854}
855
856func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
857 name, _ := ci.SplitRef(p.Ref)
858 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
859 if err != nil {
860 return nil, nil, err
861 }
862 refProtected := access.BranchProtected(patterns, name)
863 env := map[string]string{}
864 var mask []string
865 if repo.OwnerKind == "group" {
866 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
867 if err != nil {
868 return nil, nil, err
869 }
870 for k, v := range groupSecrets {
871 env[k] = v
872 mask = append(mask, v)
873 }
874 }
875 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
876 if err != nil {
877 return nil, nil, err
878 }
879 for k, v := range repoSecrets {
880 env[k] = v
881 mask = append(mask, v)
882 }
883 for k, v := range job.Variables {
884 env[k] = v
885 }
886 base := strings.TrimRight(r.BaseURL, "/")
887 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
888 var info *ci.MRInfo
889 if p.MRID != 0 {
890 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
891 target := ""
892 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
893 target = strings.TrimSpace(string(out))
894 }
895 info = &ci.MRInfo{
896 ID: m.ID, IID: m.Number, Title: m.Title,
897 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
898 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
899 TargetSHA: target,
900 }
901 }
902 }
903 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
904 pre := ci.Variables(ci.Ref{
905 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
906 DefaultBranch: repo.DefaultBranch, Source: p.Source,
907 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
908 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
909 })
910 for k, v := range pre {
911 env[k] = v
912 }
913 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
914 env["CI_JOB_NAME"] = job.Name
915 env["CI_JOB_STAGE"] = job.Stage
916 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
917 env["CI_JOB_TOKEN"] = token
918 if job.Environment != "" {
919 env["CI_ENVIRONMENT_NAME"] = job.Environment
920 if job.EnvironmentURL != "" {
921 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
922 }
923 }
924 for k, v := range job.Variables {
925 if strings.HasPrefix(k, "CI_NODE_") {
926 env[k] = v
927 }
928 }
929 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
930 // en safe.directory er alsnog bij komt.
931 markWorkspaceSafe(env)
932 JobGitAuth(env, base+"/"+repo.FullPath(), token)
933 mask = append(mask, token)
934 return env, mask, nil
935}
936
937// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
938// job-token. De header geldt alleen voor de URL van dit project (met en
939// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
940// het in elk image, zonder bestanden in de container.
941func JobGitAuth(env map[string]string, projectURL, token string) {
942 if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
943 return
944 }
945 n := 0
946 if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
947 n = v
948 }
949 for _, u := range []string{projectURL, projectURL + ".git"} {
950 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
951 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
952 n++
953 }
954 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
955}
956
957// markWorkspaceSafe zet safe.directory voor elke job.
958// De checkout is van de host-uid; de container-user is dat niet.
959// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
960// * of /workspace zet, blijft zoals hij is.
961func markWorkspaceSafe(env map[string]string) {
962 const key = "safe.directory"
963 n := 0
964 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
965 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
966 n = v
967 }
968 }
969 for i := 0; i < n; i++ {
970 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
971 continue
972 }
973 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
974 case "*", "/workspace":
975 return
976 }
977 }
978 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
979 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
980 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
981}
982
983func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
984 netName := fmt.Sprintf("bluegit-%d", job.ID)
985 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
986 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
987 }
988 var names []string
989 cleanup := func() {
990 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
991 defer cancel()
992 for _, name := range names {
993 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
994 }
995 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
996 }
997 for i, svc := range job.Services {
998 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
999 args := []string{"run", "-d", "--name", cname, "--network", netName}
1000 if svc.Alias != "" {
1001 args = append(args, "--network-alias", svc.Alias)
1002 }
1003 cmdEnv := append([]string{}, os.Environ()...)
1004 for k, v := range svc.Variables {
1005 args = append(args, "-e", k)
1006 cmdEnv = append(cmdEnv, k+"="+v)
1007 }
1008 if len(svc.Entrypoint) > 0 {
1009 args = append(args, "--entrypoint", svc.Entrypoint[0])
1010 }
1011 args = append(args, svc.Name)
1012 if len(svc.Entrypoint) > 1 {
1013 args = append(args, svc.Entrypoint[1:]...)
1014 }
1015 args = append(args, svc.Command...)
1016 cmd := exec.CommandContext(ctx, r.Docker, args...)
1017 cmd.Env = cmdEnv
1018 var stderr bytes.Buffer
1019 cmd.Stderr = &stderr
1020 if err := cmd.Run(); err != nil {
1021 cleanup()
1022 msg := strings.TrimSpace(stderr.String())
1023 if msg == "" {
1024 msg = err.Error()
1025 }
1026 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
1027 }
1028 names = append(names, cname)
1029 }
1030 return netName, cleanup, nil
1031}
1032
1033func shouldRetry(job store.Job, kind string) bool {
1034 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
1035 return false
1036 }
1037 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
1038 return true
1039 }
1040 if containsStr(job.RetryWhen, kind) {
1041 return true
1042 }
1043 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
1044}
1045
1046func classifyFailure(err error) string {
1047 msg := err.Error()
1048 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
1049 return "runner_system_failure"
1050 }
1051 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
1052 return "job_execution_timeout"
1053 }
1054 return "script_failure"
1055}
1056
1057func containsStr(list []string, s string) bool {
1058 for _, item := range list {
1059 if item == s {
1060 return true
1061 }
1062 }
1063 return false
1064}
1065
1066func safeSegment(name string) string {
1067 var b strings.Builder
1068 for _, r := range name {
1069 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1070 b.WriteRune(r)
1071 } else {
1072 b.WriteByte('_')
1073 }
1074 }
1075 if b.Len() == 0 {
1076 return "job"
1077 }
1078 return b.String()
1079}
1080
1081func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
1082 if len(job.CachePaths) == 0 {
1083 return nil
1084 }
1085 root := r.cacheDir(repoID, job.CacheKey)
1086 for _, rel := range job.CachePaths {
1087 src := filepath.Join(root, filepath.FromSlash(rel))
1088 if _, err := os.Stat(src); err != nil {
1089 continue
1090 }
1091 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
1092 return err
1093 }
1094 }
1095 return nil
1096}
1097
1098func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1099 if len(job.CachePaths) == 0 {
1100 return nil
1101 }
1102 root := r.cacheDir(repoID, job.CacheKey)
1103 for _, rel := range job.CachePaths {
1104 src := filepath.Join(work, filepath.FromSlash(rel))
1105 info, err := os.Stat(src)
1106 if err != nil || info.IsDir() {
1107 continue
1108 }
1109 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1110 return err
1111 }
1112 }
1113 return nil
1114}
1115
1116func (r *Runner) cacheDir(repoID int64, key string) string {
1117 if key == "" {
1118 key = "default"
1119 }
1120 var b strings.Builder
1121 for _, r := range key {
1122 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1123 b.WriteRune(r)
1124 }
1125 }
1126 safe := b.String()
1127 if safe == "" {
1128 safe = "default"
1129 }
1130 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1131}
1132
1133func stamp(line string) string {
1134 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1135}
1136
1137type logSink struct {
1138 mu sync.Mutex
1139 store *store.Store
1140 jobID int64
1141 out func(string)
1142 secrets []string
1143 partial string
1144 current string
1145 opened time.Time
1146}
1147
1148func (l *logSink) Write(p []byte) (int, error) {
1149 l.mu.Lock()
1150 defer l.mu.Unlock()
1151 l.partial += string(p)
1152 for {
1153 i := strings.IndexByte(l.partial, '\n')
1154 if i < 0 {
1155 break
1156 }
1157 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1158 l.partial = l.partial[i+1:]
1159 }
1160 return len(p), nil
1161}
1162
1163func (l *logSink) flush() {
1164 l.mu.Lock()
1165 defer l.mu.Unlock()
1166 if strings.TrimSpace(l.partial) != "" {
1167 l.emit(l.partial)
1168 }
1169 l.partial = ""
1170}
1171
1172func (l *logSink) emit(line string) {
1173 line = ci.Mask(line, l.secrets)
1174 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1175 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1176 dur := int(time.Since(l.opened).Seconds())
1177 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1178 l.current = ""
1179 }
1180 if strings.HasPrefix(line, "section_start:") {
1181 l.current = line
1182 l.opened = time.Now()
1183 }
1184 }
1185 if l.out != nil {
1186 l.out(stamp(line))
1187 return
1188 }
1189 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1190}
1191
1192func (l *logSink) section(name, title string) {
1193 l.emit("section_start:" + name + ":" + title)
1194}
1195
1196func (l *logSink) closeSection() {
1197 if l.current == "" {
1198 return
1199 }
1200 name := "prepare"
1201 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1202 name = parts[0]
1203 }
1204 l.emit("section_end:" + name + ":done")
1205}
1206
1207func copyFile(src, dst string) error {
1208 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1209 return err
1210 }
1211 in, err := os.Open(src)
1212 if err != nil {
1213 return err
1214 }
1215 defer in.Close()
1216 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1217 if err != nil {
1218 return err
1219 }
1220 defer out.Close()
1221 _, err = io.Copy(out, in)
1222 return err
1223}
1224
1225func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1226 if bin == "" {
1227 bin = "git"
1228 }
1229 cmd := exec.CommandContext(ctx, bin, args...)
1230 var stdout, stderr bytes.Buffer
1231 cmd.Stdout = &stdout
1232 cmd.Stderr = &stderr
1233 if err := cmd.Run(); err != nil {
1234 msg := strings.TrimSpace(stderr.String())
1235 if msg == "" {
1236 msg = err.Error()
1237 }
1238 return "", fmt.Errorf("%s", msg)
1239 }
1240 return stdout.String(), nil
1241}
1242
1243// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1244// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1245// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1246func cloneArgs(bare, work string) []string {
1247 return []string{"clone", "--no-hardlinks", bare, work}
1248}