BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 31.2 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strconv"
15 "strings"
16 "sync"
17 "time"
18
19 "gitlab.com/Timfa/bluegit/internal/access"
20 "gitlab.com/Timfa/bluegit/internal/ci"
21 "gitlab.com/Timfa/bluegit/internal/gitrepo"
22 "gitlab.com/Timfa/bluegit/internal/images"
23 "gitlab.com/Timfa/bluegit/internal/store"
24 "gitlab.com/Timfa/bluegit/internal/webhook"
25)
26
27// Runner plant jobs en stuurt het resultaat door naar webhooks.
28type Runner struct {
29 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
30 Hold func(store.Pipeline) bool
31 Store *store.Store
32 GitBin string
33 Docker string
34 Storage string
35 BaseURL string
36 Network string
37 Name string
38 Timeout time.Duration
39 Log *slog.Logger
40 OnEvent func(repoID int64, ev webhook.Event)
41 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
42 Presets map[string]images.Override
43 slots chan struct{}
44 mu sync.RWMutex
45 cancelMu sync.Mutex
46 cancel map[int64]context.CancelFunc
47}
48
49// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
50func (r *Runner) Interrupt(ids []int64) {
51 r.cancelMu.Lock()
52 defer r.cancelMu.Unlock()
53 for _, id := range ids {
54 if c := r.cancel[id]; c != nil {
55 c()
56 }
57 }
58}
59
60func (r *Runner) track(id int64, cancel context.CancelFunc) {
61 r.cancelMu.Lock()
62 defer r.cancelMu.Unlock()
63 if r.cancel == nil {
64 r.cancel = map[int64]context.CancelFunc{}
65 }
66 r.cancel[id] = cancel
67}
68
69func (r *Runner) untrack(id int64) {
70 r.cancelMu.Lock()
71 defer r.cancelMu.Unlock()
72 delete(r.cancel, id)
73}
74
75// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
76func (r *Runner) SetStorage(path string) {
77 r.mu.Lock()
78 r.Storage = path
79 r.mu.Unlock()
80}
81
82func (r *Runner) storage() string {
83 r.mu.RLock()
84 defer r.mu.RUnlock()
85 return r.Storage
86}
87
88// Loop blijft jobs oppakken tot ctx afloopt.
89func (r *Runner) Loop(ctx context.Context, concurrency int) {
90 if concurrency < 1 {
91 concurrency = 1
92 }
93 r.slots = make(chan struct{}, concurrency)
94 tick := time.NewTicker(2 * time.Second)
95 defer tick.Stop()
96 for {
97 select {
98 case <-ctx.Done():
99 return
100 case <-tick.C:
101 r.Tick(ctx)
102 }
103 }
104}
105
106// Tick bekijkt alle open pipelines één keer.
107// Zonder opslagpad gebeurt er niets.
108func (r *Runner) Tick(ctx context.Context) {
109 if r.storage() == "" {
110 return
111 }
112 pipes, err := r.Store.ActivePipelines(ctx)
113 if err != nil {
114 r.Log.Error("pipelines", "err", err)
115 return
116 }
117 for _, p := range pipes {
118 r.schedule(ctx, p)
119 }
120 r.reapRemote(ctx)
121}
122
123func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
124 r.settle(ctx, p.ID)
125 if r.Hold != nil && r.Hold(p) {
126 return // wacht op de self-update
127 }
128 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
129 if err != nil {
130 r.Log.Error("jobs", "err", err)
131 return
132 }
133 specs := make([]ci.Job, len(jobs))
134 status := map[string]string{}
135 for i, j := range jobs {
136 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
137 status[j.Name] = j.Status
138 }
139 tierCache := map[int64][]Tier{}
140 for _, j := range jobs {
141 if j.Status != "queued" && j.Status != "manual" {
142 continue
143 }
144 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
145 case "skip":
146 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
147 status[j.Name] = "skipped"
148 case "fail":
149 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
150 status[j.Name] = "failed"
151 case "ready":
152 if j.Status != "queued" {
153 continue
154 }
155 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
156 continue
157 }
158 // Meer jobs dan ci.concurrency blijven queued, met een plek
159 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
160 free := r.acquireSlot()
161 if tiers := r.tiersFor(ctx, tierCache, j.RepoID); len(tiers) > 1 {
162 jr := r.Store.JobRun(ctx, j.ID)
163 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
164 if !step.Builtin {
165 if free {
166 r.releaseSlot()
167 }
168 if step.Moved {
169 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
170 }
171 continue
172 }
173 }
174 if !free {
175 continue
176 }
177 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
178 if err != nil || !ok {
179 r.releaseSlot()
180 continue
181 }
182 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
183 _ = r.Store.ClearJobOffer(ctx, j.ID)
184 status[j.Name] = "running"
185 job := j
186 pipe := p
187 go r.execute(ctx, pipe, job)
188 }
189 }
190}
191
192// settle slaat jobs over die door een harde mislukking of een
193// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
194// pipelinestatus al op failed was gezet.
195func (r *Runner) settle(ctx context.Context, pipelineID int64) {
196 for {
197 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
198 if err != nil {
199 if r.Log != nil {
200 r.Log.Error("jobs", "err", err)
201 }
202 return
203 }
204 specs := make([]ci.Job, len(jobs))
205 status := map[string]string{}
206 for i, j := range jobs {
207 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
208 status[j.Name] = j.Status
209 }
210 moved := false
211 for _, j := range jobs {
212 if j.Status != "queued" && j.Status != "manual" {
213 continue
214 }
215 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
216 case "skip":
217 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
218 r.Log.Error("skip", "err", err)
219 continue
220 }
221 status[j.Name] = "skipped"
222 moved = true
223 case "fail":
224 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
225 r.Log.Error("fail", "err", err)
226 continue
227 }
228 status[j.Name] = "failed"
229 moved = true
230 }
231 }
232 if !moved {
233 return
234 }
235 }
236}
237
238// BuiltinName is de naam van de ingebouwde runner in de job-UI.
239func (r *Runner) BuiltinName() string {
240 if r.Name != "" {
241 return r.Name
242 }
243 return "BlueGit runner"
244}
245
246func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64) []Tier {
247 if t, ok := cache[repoID]; ok {
248 return t
249 }
250 var tiers []Tier
251 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
252 tiers, _ = Tiers(ctx, r.Store, repo)
253 }
254 cache[repoID] = tiers
255 return tiers
256}
257
258func indexOf(jobs []store.Job, name string) int {
259 for i, j := range jobs {
260 if j.Name == name {
261 return i
262 }
263 }
264 return 0
265}
266
267func (r *Runner) acquireSlot() bool {
268 if r.slots == nil {
269 return true
270 }
271 select {
272 case r.slots <- struct{}{}:
273 return true
274 default:
275 return false
276 }
277}
278
279func (r *Runner) releaseSlot() {
280 if r.slots == nil {
281 return
282 }
283 select {
284 case <-r.slots:
285 default:
286 }
287}
288
289func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
290 defer r.releaseSlot()
291 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
292 base, stop := context.WithCancel(ctx)
293 r.track(job.ID, stop)
294 defer func() {
295 r.untrack(job.ID)
296 stop()
297 }()
298 runCtx := base
299 limit := r.Timeout
300 if job.Timeout > 0 {
301 limit = job.Timeout
302 }
303 if limit > 0 {
304 var cancel context.CancelFunc
305 runCtx, cancel = context.WithTimeout(base, limit)
306 defer cancel()
307 }
308 mask, err := r.runOne(runCtx, p, job)
309 status := "success"
310 extra := ""
311 kind := ""
312 switch {
313 case err == nil:
314 case ctx.Err() == nil && base.Err() != nil:
315 status = "canceled"
316 extra = stamp("canceled because a newer pipeline started for the same ref")
317 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
318 status = "failed"
319 kind = "job_execution_timeout"
320 extra = stamp(ci.Mask(err.Error(), mask))
321 default:
322 status = "failed"
323 kind = classifyFailure(err)
324 extra = stamp(ci.Mask(err.Error(), mask))
325 }
326 r.complete(p, job, before, status, extra, kind)
327}
328
329// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
330// en webhook. Gedeeld door de ingebouwde en de aparte runner.
331func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
332 if status == "failed" && shouldRetry(job, kind) {
333 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
334 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
335 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
336 r.Log.Error("job opnieuw", "err", ferr)
337 } else if ferr == nil {
338 r.settle(context.Background(), p.ID)
339 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
340 r.Log.Error("pipelinestatus", "err", rerr)
341 }
342 return
343 }
344 }
345 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
346 r.Log.Error("job afronden", "err", ferr)
347 }
348 r.settle(context.Background(), p.ID)
349 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
350 if rerr != nil {
351 r.Log.Error("pipelinestatus", "err", rerr)
352 return
353 }
354 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
355 kind := webhook.EventPipelineSuccess
356 word := "geslaagd"
357 if final == "failed" {
358 kind = webhook.EventPipelineFailure
359 word = "mislukt"
360 }
361 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
362 if err != nil {
363 return
364 }
365 branch, _ := gitrepo.BranchOfRef(p.Ref)
366 if branch == "" {
367 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
368 }
369 r.OnEvent(p.RepoID, webhook.Event{
370 Type: kind,
371 Project: repo.FullPath(),
372 Branch: branch,
373 Commit: p.CommitSHA,
374 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
375 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
376 PipelineID: fmt.Sprint(p.ID),
377 })
378 }
379}
380
381func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
382 repo, err := r.Store.RepoByID(ctx, p.RepoID)
383 if err != nil {
384 return nil, err
385 }
386 sink := &logSink{store: r.Store, jobID: job.ID}
387 sink.section("prepare", "Preparing the docker executor")
388 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
389 _ = os.RemoveAll(work)
390 if err := os.MkdirAll(work, 0o755); err != nil {
391 sink.closeSection()
392 return nil, err
393 }
394 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
395 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
396 sink.closeSection()
397 return nil, fmt.Errorf("clone: %w", err)
398 }
399 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
400 sink.closeSection()
401 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
402 }
403 if err := r.restoreCache(repo.ID, job, work); err != nil {
404 sink.closeSection()
405 return nil, err
406 }
407 if err := r.materialize(ctx, p, job, work); err != nil {
408 sink.closeSection()
409 return nil, err
410 }
411 scriptPath := filepath.Join(work, ".bluegit-job.sh")
412 host, _ := os.Hostname()
413 script := shellScript(job, placeNote{
414 Runner: r.Name, Host: host, WorkDir: "/workspace",
415 PipelineID: p.ID, JobID: job.ID,
416 })
417 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
418 sink.closeSection()
419 return nil, err
420 }
421 token, err := r.Store.CreateJobToken(ctx, job.ID)
422 if err != nil {
423 sink.closeSection()
424 return nil, err
425 }
426 env, mask, err := r.environment(ctx, repo, p, job, token)
427 if err != nil {
428 sink.closeSection()
429 return nil, err
430 }
431 sink.secrets = mask
432 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
433 return mask, err
434 }
435 if err := r.collect(ctx, job, work); err != nil {
436 return mask, err
437 }
438 if err := r.saveCache(repo.ID, job, work); err != nil {
439 return mask, err
440 }
441 return mask, nil
442}
443
444// execContainer haalt het image op, start services en draait het
445// jobscript in de container. De sectie prepare moet open staan.
446func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
447 ref, lines, rerr := r.resolveImage(ctx, job.Image)
448 for _, line := range lines {
449 sink.emit(line)
450 }
451 if rerr != nil {
452 sink.closeSection()
453 return rerr
454 }
455 job.Image = ref
456 job.Services = append([]store.Service(nil), job.Services...)
457 for i := range job.Services {
458 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
459 for _, line := range slines {
460 sink.emit(line)
461 }
462 if serr != nil {
463 sink.closeSection()
464 return serr
465 }
466 job.Services[i].Name = sref
467 }
468 network := r.Network
469 if network == "" {
470 network = "bridge"
471 }
472 if len(job.Services) > 0 {
473 net, cleanup, serr := r.startServices(ctx, job)
474 if serr != nil {
475 sink.closeSection()
476 return serr
477 }
478 defer cleanup()
479 network = net
480 }
481 sink.closeSection()
482 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
483 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
484 // staat de waarde niet in de proceslijst.
485 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
486 cmdEnv := append([]string{}, os.Environ()...)
487 for k, v := range env {
488 args = append(args, "-e", k)
489 cmdEnv = append(cmdEnv, k+"="+v)
490 }
491 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
492 cmd := exec.CommandContext(ctx, r.Docker, args...)
493 cmd.Env = cmdEnv
494 cmd.Stdout = sink
495 cmd.Stderr = sink
496 err := cmd.Run()
497 sink.flush()
498 if err != nil {
499 return fmt.Errorf("container ended with an error: %w", err)
500 }
501 return nil
502}
503
504// placeNote zijn de vaste regels in de sectie Preparing environment.
505type placeNote struct {
506 Runner string
507 Host string
508 WorkDir string
509 PipelineID int64
510 JobID int64
511}
512
513func (n placeNote) lines() []string {
514 runner := strings.TrimSpace(n.Runner)
515 if runner == "" {
516 runner = "Steve"
517 }
518 host := strings.TrimSpace(n.Host)
519 if host == "" {
520 host = "localhost"
521 }
522 dir := strings.TrimSpace(n.WorkDir)
523 if dir == "" {
524 dir = "/workspace"
525 }
526 return []string{
527 fmt.Sprintf("Running on %s via %s...", runner, host),
528 fmt.Sprintf("Working directory: %s", dir),
529 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
530 }
531}
532
533func shSingle(s string) string {
534 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
535}
536
537func shellScript(job store.Job, note placeNote) string {
538 var b strings.Builder
539 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
540 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
541 for _, line := range note.lines() {
542 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
543 }
544 if strings.TrimSpace(job.Before) != "" {
545 b.WriteString("(\nset -e\n")
546 b.WriteString(job.Before)
547 if !strings.HasSuffix(job.Before, "\n") {
548 b.WriteByte('\n')
549 }
550 b.WriteString(")\nbrc=$?\n")
551 } else {
552 b.WriteString("brc=0\n")
553 }
554 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
555 b.WriteString("echo \"section_start:script:Running script\"\n")
556 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
557 b.WriteString(job.Script)
558 if !strings.HasSuffix(job.Script, "\n") {
559 b.WriteByte('\n')
560 }
561 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
562 b.WriteString("echo \"section_end:script:Running script\"\n")
563 b.WriteString("echo \"section_start:after:After script\"\n")
564 if strings.TrimSpace(job.After) != "" {
565 b.WriteString("(\nset -e\n")
566 b.WriteString(job.After)
567 if !strings.HasSuffix(job.After, "\n") {
568 b.WriteByte('\n')
569 }
570 b.WriteString(")\n")
571 }
572 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
573 return b.String()
574}
575
576// resolveImage kiest het image waarmee de container start.
577// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
578func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
579 image = strings.TrimSpace(image)
580 set, err := images.Effective(r.Presets)
581 if err != nil {
582 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
583 }
584 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
585 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
586 if p, ok := images.Find(set, image); ok {
587 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
588 }
589 return image, dockerImageLines(image, "", nil), missing
590 }
591 p, ok := images.Find(set, image)
592 if !ok {
593 id, digests := r.inspectImage(ctx, image)
594 return image, dockerImageLines(image, id, digests), nil
595 }
596 ref, built, err := p.Ensure(ctx, r.Docker)
597 if err != nil {
598 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
599 }
600 var lines []string
601 if built {
602 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
603 }
604 id, digests := r.inspectImage(ctx, ref)
605 var baseDigests []string
606 if ref != p.From {
607 _, baseDigests = r.inspectImage(ctx, p.From)
608 }
609 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
610 return ref, lines, nil
611}
612
613func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
614 var lines []string
615 if preset != "" {
616 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
617 }
618 if ref == "" {
619 return lines
620 }
621 lines = append(lines, dockerImageLines(ref, id, digests)...)
622 if preset != "" && ref != base {
623 digest := matchingDigest(base, baseDigests)
624 if digest == "" && len(baseDigests) > 0 {
625 digest = strings.TrimSpace(baseDigests[0])
626 }
627 if digest != "" {
628 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
629 } else {
630 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
631 }
632 }
633 return lines
634}
635
636func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
637 if strings.TrimSpace(image) == "" || r.Docker == "" {
638 return "", nil
639 }
640 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
641 if err != nil {
642 return "", nil
643 }
644 return parseImageInspect(out)
645}
646
647func parseImageInspect(out string) (string, []string) {
648 lines := strings.Split(out, "\n")
649 id := ""
650 if len(lines) > 0 {
651 id = strings.TrimSpace(lines[0])
652 }
653 var digests []string
654 for _, line := range lines[1:] {
655 line = strings.TrimSpace(line)
656 if line != "" {
657 digests = append(digests, line)
658 }
659 }
660 return id, digests
661}
662
663func dockerImageLines(image, id string, digests []string) []string {
664 image = strings.TrimSpace(image)
665 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
666 id = strings.TrimSpace(id)
667 digest := matchingDigest(image, digests)
668 switch {
669 case id != "" && digest != "":
670 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
671 case digest != "":
672 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
673 case id != "":
674 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
675 }
676 return lines
677}
678
679func matchingDigest(image string, digests []string) string {
680 name := imageName(image)
681 fallback := ""
682 for _, digest := range digests {
683 digest = strings.TrimSpace(digest)
684 at := strings.LastIndex(digest, "@")
685 if digest == "" || at <= 0 {
686 continue
687 }
688 if fallback == "" {
689 fallback = digest
690 }
691 repo := digest[:at]
692 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
693 return digest
694 }
695 }
696 return fallback
697}
698
699func imageName(image string) string {
700 image = strings.TrimSpace(image)
701 if i := strings.LastIndex(image, "@"); i >= 0 {
702 image = image[:i]
703 }
704 slash := strings.LastIndex(image, "/")
705 name := image
706 if slash >= 0 {
707 name = image[slash+1:]
708 }
709 if i := strings.Index(name, ":"); i >= 0 {
710 name = name[:i]
711 }
712 return name
713}
714
715func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
716 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
717 if err != nil {
718 return err
719 }
720 wanted := map[string]bool{}
721 if job.NeedsSet {
722 for _, n := range job.Needs {
723 wanted[n] = true
724 }
725 } else {
726 stageOrder := map[string]int{}
727 n := 0
728 for _, j := range jobs {
729 if _, ok := stageOrder[j.Stage]; !ok {
730 stageOrder[j.Stage] = n
731 n++
732 }
733 }
734 my := stageOrder[job.Stage]
735 for _, j := range jobs {
736 if stageOrder[j.Stage] < my {
737 wanted[j.Name] = true
738 }
739 }
740 }
741 for _, j := range jobs {
742 if !wanted[j.Name] || j.Status != "success" {
743 continue
744 }
745 for _, rel := range j.ArtifactPaths {
746 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
747 if err != nil {
748 continue
749 }
750 dst := filepath.Join(work, filepath.FromSlash(rel))
751 if err := copyFile(src, dst); err != nil {
752 return err
753 }
754 }
755 }
756 return nil
757}
758
759func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
760 var expires *time.Time
761 if job.ArtifactExpire > 0 {
762 t := time.Now().Add(job.ArtifactExpire)
763 expires = &t
764 }
765 for _, rel := range job.ArtifactPaths {
766 src := filepath.Join(work, filepath.FromSlash(rel))
767 info, err := os.Stat(src)
768 if err != nil || info.IsDir() {
769 return fmt.Errorf("artifact %s ontbreekt", rel)
770 }
771 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
772 if err := copyFile(src, dst); err != nil {
773 return err
774 }
775 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
776 return err
777 }
778 }
779 return nil
780}
781
782func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
783 name, _ := ci.SplitRef(p.Ref)
784 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
785 if err != nil {
786 return nil, nil, err
787 }
788 refProtected := access.BranchProtected(patterns, name)
789 env := map[string]string{}
790 var mask []string
791 if repo.OwnerKind == "group" {
792 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
793 if err != nil {
794 return nil, nil, err
795 }
796 for k, v := range groupSecrets {
797 env[k] = v
798 mask = append(mask, v)
799 }
800 }
801 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
802 if err != nil {
803 return nil, nil, err
804 }
805 for k, v := range repoSecrets {
806 env[k] = v
807 mask = append(mask, v)
808 }
809 for k, v := range job.Variables {
810 env[k] = v
811 }
812 base := strings.TrimRight(r.BaseURL, "/")
813 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
814 var info *ci.MRInfo
815 if p.MRID != 0 {
816 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
817 target := ""
818 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
819 target = strings.TrimSpace(string(out))
820 }
821 info = &ci.MRInfo{
822 ID: m.ID, IID: m.Number, Title: m.Title,
823 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
824 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
825 TargetSHA: target,
826 }
827 }
828 }
829 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
830 pre := ci.Variables(ci.Ref{
831 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
832 DefaultBranch: repo.DefaultBranch, Source: p.Source,
833 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
834 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
835 })
836 for k, v := range pre {
837 env[k] = v
838 }
839 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
840 env["CI_JOB_NAME"] = job.Name
841 env["CI_JOB_STAGE"] = job.Stage
842 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
843 env["CI_JOB_TOKEN"] = token
844 if job.Environment != "" {
845 env["CI_ENVIRONMENT_NAME"] = job.Environment
846 if job.EnvironmentURL != "" {
847 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
848 }
849 }
850 for k, v := range job.Variables {
851 if strings.HasPrefix(k, "CI_NODE_") {
852 env[k] = v
853 }
854 }
855 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
856 // en safe.directory er alsnog bij komt.
857 markWorkspaceSafe(env)
858 mask = append(mask, token)
859 return env, mask, nil
860}
861
862// markWorkspaceSafe zet safe.directory voor elke job.
863// De checkout is van de host-uid; de container-user is dat niet.
864// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
865// * of /workspace zet, blijft zoals hij is.
866func markWorkspaceSafe(env map[string]string) {
867 const key = "safe.directory"
868 n := 0
869 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
870 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
871 n = v
872 }
873 }
874 for i := 0; i < n; i++ {
875 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
876 continue
877 }
878 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
879 case "*", "/workspace":
880 return
881 }
882 }
883 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
884 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
885 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
886}
887
888func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
889 netName := fmt.Sprintf("bluegit-%d", job.ID)
890 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
891 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
892 }
893 var names []string
894 cleanup := func() {
895 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
896 defer cancel()
897 for _, name := range names {
898 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
899 }
900 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
901 }
902 for i, svc := range job.Services {
903 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
904 args := []string{"run", "-d", "--name", cname, "--network", netName}
905 if svc.Alias != "" {
906 args = append(args, "--network-alias", svc.Alias)
907 }
908 cmdEnv := append([]string{}, os.Environ()...)
909 for k, v := range svc.Variables {
910 args = append(args, "-e", k)
911 cmdEnv = append(cmdEnv, k+"="+v)
912 }
913 if len(svc.Entrypoint) > 0 {
914 args = append(args, "--entrypoint", svc.Entrypoint[0])
915 }
916 args = append(args, svc.Name)
917 if len(svc.Entrypoint) > 1 {
918 args = append(args, svc.Entrypoint[1:]...)
919 }
920 args = append(args, svc.Command...)
921 cmd := exec.CommandContext(ctx, r.Docker, args...)
922 cmd.Env = cmdEnv
923 var stderr bytes.Buffer
924 cmd.Stderr = &stderr
925 if err := cmd.Run(); err != nil {
926 cleanup()
927 msg := strings.TrimSpace(stderr.String())
928 if msg == "" {
929 msg = err.Error()
930 }
931 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
932 }
933 names = append(names, cname)
934 }
935 return netName, cleanup, nil
936}
937
938func shouldRetry(job store.Job, kind string) bool {
939 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
940 return false
941 }
942 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
943 return true
944 }
945 if containsStr(job.RetryWhen, kind) {
946 return true
947 }
948 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
949}
950
951func classifyFailure(err error) string {
952 msg := err.Error()
953 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
954 return "runner_system_failure"
955 }
956 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
957 return "job_execution_timeout"
958 }
959 return "script_failure"
960}
961
962func containsStr(list []string, s string) bool {
963 for _, item := range list {
964 if item == s {
965 return true
966 }
967 }
968 return false
969}
970
971func safeSegment(name string) string {
972 var b strings.Builder
973 for _, r := range name {
974 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
975 b.WriteRune(r)
976 } else {
977 b.WriteByte('_')
978 }
979 }
980 if b.Len() == 0 {
981 return "job"
982 }
983 return b.String()
984}
985
986func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
987 if len(job.CachePaths) == 0 {
988 return nil
989 }
990 root := r.cacheDir(repoID, job.CacheKey)
991 for _, rel := range job.CachePaths {
992 src := filepath.Join(root, filepath.FromSlash(rel))
993 if _, err := os.Stat(src); err != nil {
994 continue
995 }
996 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
997 return err
998 }
999 }
1000 return nil
1001}
1002
1003func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1004 if len(job.CachePaths) == 0 {
1005 return nil
1006 }
1007 root := r.cacheDir(repoID, job.CacheKey)
1008 for _, rel := range job.CachePaths {
1009 src := filepath.Join(work, filepath.FromSlash(rel))
1010 info, err := os.Stat(src)
1011 if err != nil || info.IsDir() {
1012 continue
1013 }
1014 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1015 return err
1016 }
1017 }
1018 return nil
1019}
1020
1021func (r *Runner) cacheDir(repoID int64, key string) string {
1022 if key == "" {
1023 key = "default"
1024 }
1025 var b strings.Builder
1026 for _, r := range key {
1027 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1028 b.WriteRune(r)
1029 }
1030 }
1031 safe := b.String()
1032 if safe == "" {
1033 safe = "default"
1034 }
1035 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1036}
1037
1038func stamp(line string) string {
1039 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1040}
1041
1042type logSink struct {
1043 mu sync.Mutex
1044 store *store.Store
1045 jobID int64
1046 out func(string)
1047 secrets []string
1048 partial string
1049 current string
1050 opened time.Time
1051}
1052
1053func (l *logSink) Write(p []byte) (int, error) {
1054 l.mu.Lock()
1055 defer l.mu.Unlock()
1056 l.partial += string(p)
1057 for {
1058 i := strings.IndexByte(l.partial, '\n')
1059 if i < 0 {
1060 break
1061 }
1062 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1063 l.partial = l.partial[i+1:]
1064 }
1065 return len(p), nil
1066}
1067
1068func (l *logSink) flush() {
1069 l.mu.Lock()
1070 defer l.mu.Unlock()
1071 if strings.TrimSpace(l.partial) != "" {
1072 l.emit(l.partial)
1073 }
1074 l.partial = ""
1075}
1076
1077func (l *logSink) emit(line string) {
1078 line = ci.Mask(line, l.secrets)
1079 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1080 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1081 dur := int(time.Since(l.opened).Seconds())
1082 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1083 l.current = ""
1084 }
1085 if strings.HasPrefix(line, "section_start:") {
1086 l.current = line
1087 l.opened = time.Now()
1088 }
1089 }
1090 if l.out != nil {
1091 l.out(stamp(line))
1092 return
1093 }
1094 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1095}
1096
1097func (l *logSink) section(name, title string) {
1098 l.emit("section_start:" + name + ":" + title)
1099}
1100
1101func (l *logSink) closeSection() {
1102 if l.current == "" {
1103 return
1104 }
1105 name := "prepare"
1106 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1107 name = parts[0]
1108 }
1109 l.emit("section_end:" + name + ":done")
1110}
1111
1112func copyFile(src, dst string) error {
1113 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1114 return err
1115 }
1116 in, err := os.Open(src)
1117 if err != nil {
1118 return err
1119 }
1120 defer in.Close()
1121 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1122 if err != nil {
1123 return err
1124 }
1125 defer out.Close()
1126 _, err = io.Copy(out, in)
1127 return err
1128}
1129
1130func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1131 if bin == "" {
1132 bin = "git"
1133 }
1134 cmd := exec.CommandContext(ctx, bin, args...)
1135 var stdout, stderr bytes.Buffer
1136 cmd.Stdout = &stdout
1137 cmd.Stderr = &stderr
1138 if err := cmd.Run(); err != nil {
1139 msg := strings.TrimSpace(stderr.String())
1140 if msg == "" {
1141 msg = err.Error()
1142 }
1143 return "", fmt.Errorf("%s", msg)
1144 }
1145 return stdout.String(), nil
1146}
1147
1148// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1149// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1150// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1151func cloneArgs(bare, work string) []string {
1152 return []string{"clone", "--no-hardlinks", bare, work}
1153}