BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 33.2 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strconv"
15 "strings"
16 "sync"
17 "time"
18
19 "gitlab.com/Timfa/bluegit/internal/access"
20 "gitlab.com/Timfa/bluegit/internal/ci"
21 "gitlab.com/Timfa/bluegit/internal/gitrepo"
22 "gitlab.com/Timfa/bluegit/internal/images"
23 "gitlab.com/Timfa/bluegit/internal/runnercfg"
24 "gitlab.com/Timfa/bluegit/internal/store"
25 "gitlab.com/Timfa/bluegit/internal/webhook"
26)
27
28// Runner plant jobs en stuurt het resultaat door naar webhooks.
29type Runner struct {
30 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
31 Hold func(store.Pipeline) bool
32 Store *store.Store
33 GitBin string
34 Docker string
35 Storage string
36 BaseURL string
37 Network string
38 Name string
39 Timeout time.Duration
40 Log *slog.Logger
41 OnEvent func(repoID int64, ev webhook.Event)
42 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
43 Presets map[string]images.Override
44 // Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
45 Jobs *runnercfg.Tracker
46 ctl control
47 slots chan struct{}
48 mu sync.RWMutex
49 cancelMu sync.Mutex
50 cancel map[int64]context.CancelFunc
51}
52
53// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
54func (r *Runner) Interrupt(ids []int64) {
55 r.cancelMu.Lock()
56 defer r.cancelMu.Unlock()
57 for _, id := range ids {
58 if c := r.cancel[id]; c != nil {
59 c()
60 }
61 }
62}
63
64func (r *Runner) track(id int64, cancel context.CancelFunc) {
65 r.cancelMu.Lock()
66 defer r.cancelMu.Unlock()
67 if r.cancel == nil {
68 r.cancel = map[int64]context.CancelFunc{}
69 }
70 r.cancel[id] = cancel
71}
72
73func (r *Runner) untrack(id int64) {
74 r.cancelMu.Lock()
75 defer r.cancelMu.Unlock()
76 delete(r.cancel, id)
77}
78
79// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
80func (r *Runner) SetStorage(path string) {
81 r.mu.Lock()
82 r.Storage = path
83 r.mu.Unlock()
84}
85
86func (r *Runner) storage() string {
87 r.mu.RLock()
88 defer r.mu.RUnlock()
89 return r.Storage
90}
91
92// Loop blijft jobs oppakken tot ctx afloopt.
93func (r *Runner) Loop(ctx context.Context, concurrency int) {
94 if concurrency < 1 {
95 concurrency = 1
96 }
97 r.slots = make(chan struct{}, maxSlots)
98 r.mu.Lock()
99 if !r.ctl.set {
100 r.ctl.limit = concurrency
101 }
102 r.mu.Unlock()
103 tick := time.NewTicker(2 * time.Second)
104 defer tick.Stop()
105 for {
106 select {
107 case <-ctx.Done():
108 return
109 case <-tick.C:
110 r.Tick(ctx)
111 }
112 }
113}
114
115// Tick bekijkt alle open pipelines één keer.
116// Zonder opslagpad gebeurt er niets.
117func (r *Runner) Tick(ctx context.Context) {
118 if r.storage() == "" {
119 return
120 }
121 pipes, err := r.Store.ActivePipelines(ctx)
122 if err != nil {
123 r.Log.Error("pipelines", "err", err)
124 return
125 }
126 for _, p := range pipes {
127 r.schedule(ctx, p)
128 }
129 r.reapRemote(ctx)
130}
131
132func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
133 r.settle(ctx, p.ID)
134 if r.Hold != nil && r.Hold(p) {
135 return // wacht op de self-update
136 }
137 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
138 if err != nil {
139 r.Log.Error("jobs", "err", err)
140 return
141 }
142 specs := make([]ci.Job, len(jobs))
143 status := map[string]string{}
144 for i, j := range jobs {
145 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
146 status[j.Name] = j.Status
147 }
148 tierCache := map[int64][]Tier{}
149 for _, j := range jobs {
150 if j.Status != "queued" && j.Status != "manual" {
151 continue
152 }
153 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
154 case "skip":
155 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
156 status[j.Name] = "skipped"
157 case "fail":
158 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
159 status[j.Name] = "failed"
160 case "ready":
161 if j.Status != "queued" {
162 continue
163 }
164 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
165 continue
166 }
167 // Meer jobs dan ci.concurrency blijven queued, met een plek
168 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
169 free := r.acquireSlot()
170 if tiers := r.tiersFor(ctx, tierCache, j.RepoID); len(tiers) > 1 {
171 jr := r.Store.JobRun(ctx, j.ID)
172 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
173 if !step.Builtin {
174 if free {
175 r.releaseSlot()
176 }
177 if step.Moved {
178 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
179 }
180 continue
181 }
182 }
183 if !free {
184 continue
185 }
186 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
187 if err != nil || !ok {
188 r.releaseSlot()
189 continue
190 }
191 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
192 _ = r.Store.ClearJobOffer(ctx, j.ID)
193 status[j.Name] = "running"
194 job := j
195 pipe := p
196 go r.execute(ctx, pipe, job)
197 }
198 }
199}
200
201// settle slaat jobs over die door een harde mislukking of een
202// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
203// pipelinestatus al op failed was gezet.
204func (r *Runner) settle(ctx context.Context, pipelineID int64) {
205 for {
206 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
207 if err != nil {
208 if r.Log != nil {
209 r.Log.Error("jobs", "err", err)
210 }
211 return
212 }
213 specs := make([]ci.Job, len(jobs))
214 status := map[string]string{}
215 for i, j := range jobs {
216 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
217 status[j.Name] = j.Status
218 }
219 moved := false
220 for _, j := range jobs {
221 if j.Status != "queued" && j.Status != "manual" {
222 continue
223 }
224 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
225 case "skip":
226 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
227 r.Log.Error("skip", "err", err)
228 continue
229 }
230 status[j.Name] = "skipped"
231 moved = true
232 case "fail":
233 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
234 r.Log.Error("fail", "err", err)
235 continue
236 }
237 status[j.Name] = "failed"
238 moved = true
239 }
240 }
241 if !moved {
242 return
243 }
244 }
245}
246
247// BuiltinName is de naam van de ingebouwde runner in de job-UI.
248func (r *Runner) BuiltinName() string {
249 if r.Name != "" {
250 return r.Name
251 }
252 return "BlueGit runner"
253}
254
255func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64) []Tier {
256 if t, ok := cache[repoID]; ok {
257 return t
258 }
259 var tiers []Tier
260 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
261 tiers, _ = Tiers(ctx, r.Store, repo)
262 }
263 cache[repoID] = tiers
264 return tiers
265}
266
267func indexOf(jobs []store.Job, name string) int {
268 for i, j := range jobs {
269 if j.Name == name {
270 return i
271 }
272 }
273 return 0
274}
275
276func (r *Runner) acquireSlot() bool {
277 if r.Paused() {
278 return false
279 }
280 if r.slots == nil {
281 return true
282 }
283 if l := r.controls().limit; l > 0 && len(r.slots) >= l {
284 return false
285 }
286 select {
287 case r.slots <- struct{}{}:
288 return true
289 default:
290 return false
291 }
292}
293
294func (r *Runner) releaseSlot() {
295 if r.slots == nil {
296 return
297 }
298 select {
299 case <-r.slots:
300 default:
301 }
302}
303
304func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
305 defer r.releaseSlot()
306 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
307 base, stop := context.WithCancel(ctx)
308 r.track(job.ID, stop)
309 defer func() {
310 r.untrack(job.ID)
311 stop()
312 }()
313 runCtx := base
314 limit := r.Timeout
315 if job.Timeout > 0 {
316 limit = job.Timeout
317 }
318 if m := r.controls().timeout; m > 0 && (limit == 0 || time.Duration(m)*time.Minute < limit) {
319 limit = time.Duration(m) * time.Minute
320 }
321 r.trackStart(job)
322 if limit > 0 {
323 var cancel context.CancelFunc
324 runCtx, cancel = context.WithTimeout(base, limit)
325 defer cancel()
326 }
327 mask, err := r.runOne(runCtx, p, job)
328 status := "success"
329 extra := ""
330 kind := ""
331 switch {
332 case err == nil:
333 case ctx.Err() == nil && base.Err() != nil:
334 status = "canceled"
335 extra = stamp("canceled because a newer pipeline started for the same ref")
336 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
337 status = "failed"
338 kind = "job_execution_timeout"
339 extra = stamp(ci.Mask(err.Error(), mask))
340 default:
341 status = "failed"
342 kind = classifyFailure(err)
343 extra = stamp(ci.Mask(err.Error(), mask))
344 }
345 r.trackDone(job.ID, status)
346 r.complete(p, job, before, status, extra, kind)
347}
348
349// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
350// en webhook. Gedeeld door de ingebouwde en de aparte runner.
351func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
352 if status == "failed" && shouldRetry(job, kind) {
353 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
354 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
355 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
356 r.Log.Error("job opnieuw", "err", ferr)
357 } else if ferr == nil {
358 r.settle(context.Background(), p.ID)
359 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
360 r.Log.Error("pipelinestatus", "err", rerr)
361 }
362 return
363 }
364 }
365 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
366 r.Log.Error("job afronden", "err", ferr)
367 }
368 r.settle(context.Background(), p.ID)
369 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
370 if rerr != nil {
371 r.Log.Error("pipelinestatus", "err", rerr)
372 return
373 }
374 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
375 kind := webhook.EventPipelineSuccess
376 word := "geslaagd"
377 if final == "failed" {
378 kind = webhook.EventPipelineFailure
379 word = "mislukt"
380 }
381 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
382 if err != nil {
383 return
384 }
385 branch, _ := gitrepo.BranchOfRef(p.Ref)
386 if branch == "" {
387 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
388 }
389 r.OnEvent(p.RepoID, webhook.Event{
390 Type: kind,
391 Project: repo.FullPath(),
392 Branch: branch,
393 Commit: p.CommitSHA,
394 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
395 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
396 PipelineID: fmt.Sprint(p.ID),
397 })
398 }
399}
400
401func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
402 repo, err := r.Store.RepoByID(ctx, p.RepoID)
403 if err != nil {
404 return nil, err
405 }
406 sink := &logSink{store: r.Store, jobID: job.ID}
407 sink.section("prepare", "Preparing the docker executor")
408 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
409 _ = os.RemoveAll(work)
410 if err := os.MkdirAll(work, 0o755); err != nil {
411 sink.closeSection()
412 return nil, err
413 }
414 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
415 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
416 sink.closeSection()
417 return nil, fmt.Errorf("clone: %w", err)
418 }
419 // origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
420 // op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
421 if r.BaseURL != "" {
422 _, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
423 }
424 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
425 sink.closeSection()
426 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
427 }
428 if err := r.restoreCache(repo.ID, job, work); err != nil {
429 sink.closeSection()
430 return nil, err
431 }
432 if err := r.materialize(ctx, p, job, work); err != nil {
433 sink.closeSection()
434 return nil, err
435 }
436 scriptPath := filepath.Join(work, ".bluegit-job.sh")
437 host, _ := os.Hostname()
438 script := shellScript(job, placeNote{
439 Runner: r.Name, Host: host, WorkDir: "/workspace",
440 PipelineID: p.ID, JobID: job.ID,
441 })
442 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
443 sink.closeSection()
444 return nil, err
445 }
446 token, err := r.Store.CreateJobToken(ctx, job.ID)
447 if err != nil {
448 sink.closeSection()
449 return nil, err
450 }
451 env, mask, err := r.environment(ctx, repo, p, job, token)
452 if err != nil {
453 sink.closeSection()
454 return nil, err
455 }
456 sink.secrets = mask
457 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
458 return mask, err
459 }
460 if err := r.collect(ctx, job, work); err != nil {
461 return mask, err
462 }
463 if err := r.saveCache(repo.ID, job, work); err != nil {
464 return mask, err
465 }
466 return mask, nil
467}
468
469// execContainer haalt het image op, start services en draait het
470// jobscript in de container. De sectie prepare moet open staan.
471func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
472 ref, lines, rerr := r.resolveImage(ctx, job.Image)
473 for _, line := range lines {
474 sink.emit(line)
475 }
476 if rerr != nil {
477 sink.closeSection()
478 return rerr
479 }
480 job.Image = ref
481 job.Services = append([]store.Service(nil), job.Services...)
482 for i := range job.Services {
483 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
484 for _, line := range slines {
485 sink.emit(line)
486 }
487 if serr != nil {
488 sink.closeSection()
489 return serr
490 }
491 job.Services[i].Name = sref
492 }
493 network := r.Network
494 if network == "" {
495 network = "bridge"
496 }
497 if len(job.Services) > 0 {
498 net, cleanup, serr := r.startServices(ctx, job)
499 if serr != nil {
500 sink.closeSection()
501 return serr
502 }
503 defer cleanup()
504 network = net
505 }
506 sink.closeSection()
507 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
508 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
509 // staat de waarde niet in de proceslijst.
510 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
511 ctl := r.controls()
512 args = append(args, ctl.runArgs...)
513 if ctl.pull == runnercfg.PullNever {
514 args = append(args, "--pull", "never")
515 }
516 cmdEnv := append([]string{}, os.Environ()...)
517 for k, v := range env {
518 args = append(args, "-e", k)
519 cmdEnv = append(cmdEnv, k+"="+v)
520 }
521 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
522 cmd := exec.CommandContext(ctx, r.Docker, args...)
523 cmd.Env = cmdEnv
524 cmd.Stdout = sink
525 cmd.Stderr = sink
526 err := cmd.Run()
527 sink.flush()
528 if err != nil {
529 return fmt.Errorf("container ended with an error: %w", err)
530 }
531 return nil
532}
533
534// placeNote zijn de vaste regels in de sectie Preparing environment.
535type placeNote struct {
536 Runner string
537 Host string
538 WorkDir string
539 PipelineID int64
540 JobID int64
541}
542
543func (n placeNote) lines() []string {
544 runner := strings.TrimSpace(n.Runner)
545 if runner == "" {
546 runner = "Steve"
547 }
548 host := strings.TrimSpace(n.Host)
549 if host == "" {
550 host = "localhost"
551 }
552 dir := strings.TrimSpace(n.WorkDir)
553 if dir == "" {
554 dir = "/workspace"
555 }
556 return []string{
557 fmt.Sprintf("Running on %s via %s...", runner, host),
558 fmt.Sprintf("Working directory: %s", dir),
559 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
560 }
561}
562
563func shSingle(s string) string {
564 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
565}
566
567func shellScript(job store.Job, note placeNote) string {
568 var b strings.Builder
569 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
570 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
571 for _, line := range note.lines() {
572 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
573 }
574 if strings.TrimSpace(job.Before) != "" {
575 b.WriteString("(\nset -e\n")
576 b.WriteString(job.Before)
577 if !strings.HasSuffix(job.Before, "\n") {
578 b.WriteByte('\n')
579 }
580 b.WriteString(")\nbrc=$?\n")
581 } else {
582 b.WriteString("brc=0\n")
583 }
584 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
585 b.WriteString("echo \"section_start:script:Running script\"\n")
586 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
587 b.WriteString(job.Script)
588 if !strings.HasSuffix(job.Script, "\n") {
589 b.WriteByte('\n')
590 }
591 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
592 b.WriteString("echo \"section_end:script:Running script\"\n")
593 b.WriteString("echo \"section_start:after:After script\"\n")
594 if strings.TrimSpace(job.After) != "" {
595 b.WriteString("(\nset -e\n")
596 b.WriteString(job.After)
597 if !strings.HasSuffix(job.After, "\n") {
598 b.WriteByte('\n')
599 }
600 b.WriteString(")\n")
601 }
602 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
603 return b.String()
604}
605
606// resolveImage kiest het image waarmee de container start.
607// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
608func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
609 image = strings.TrimSpace(image)
610 set, err := images.Effective(r.presetSet())
611 if err != nil {
612 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
613 }
614 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
615 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
616 if p, ok := images.Find(set, image); ok {
617 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
618 }
619 return image, dockerImageLines(image, "", nil), missing
620 }
621 p, ok := images.Find(set, image)
622 if !ok {
623 if r.controls().pull == runnercfg.PullAlways {
624 _, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
625 }
626 id, digests := r.inspectImage(ctx, image)
627 return image, dockerImageLines(image, id, digests), nil
628 }
629 ref, built, err := p.Ensure(ctx, r.Docker)
630 if err != nil {
631 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
632 }
633 var lines []string
634 if built {
635 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
636 }
637 id, digests := r.inspectImage(ctx, ref)
638 var baseDigests []string
639 if ref != p.From {
640 _, baseDigests = r.inspectImage(ctx, p.From)
641 }
642 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
643 return ref, lines, nil
644}
645
646func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
647 var lines []string
648 if preset != "" {
649 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
650 }
651 if ref == "" {
652 return lines
653 }
654 lines = append(lines, dockerImageLines(ref, id, digests)...)
655 if preset != "" && ref != base {
656 digest := matchingDigest(base, baseDigests)
657 if digest == "" && len(baseDigests) > 0 {
658 digest = strings.TrimSpace(baseDigests[0])
659 }
660 if digest != "" {
661 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
662 } else {
663 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
664 }
665 }
666 return lines
667}
668
669func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
670 if strings.TrimSpace(image) == "" || r.Docker == "" {
671 return "", nil
672 }
673 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
674 if err != nil {
675 return "", nil
676 }
677 return parseImageInspect(out)
678}
679
680func parseImageInspect(out string) (string, []string) {
681 lines := strings.Split(out, "\n")
682 id := ""
683 if len(lines) > 0 {
684 id = strings.TrimSpace(lines[0])
685 }
686 var digests []string
687 for _, line := range lines[1:] {
688 line = strings.TrimSpace(line)
689 if line != "" {
690 digests = append(digests, line)
691 }
692 }
693 return id, digests
694}
695
696func dockerImageLines(image, id string, digests []string) []string {
697 image = strings.TrimSpace(image)
698 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
699 id = strings.TrimSpace(id)
700 digest := matchingDigest(image, digests)
701 switch {
702 case id != "" && digest != "":
703 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
704 case digest != "":
705 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
706 case id != "":
707 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
708 }
709 return lines
710}
711
712func matchingDigest(image string, digests []string) string {
713 name := imageName(image)
714 fallback := ""
715 for _, digest := range digests {
716 digest = strings.TrimSpace(digest)
717 at := strings.LastIndex(digest, "@")
718 if digest == "" || at <= 0 {
719 continue
720 }
721 if fallback == "" {
722 fallback = digest
723 }
724 repo := digest[:at]
725 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
726 return digest
727 }
728 }
729 return fallback
730}
731
732func imageName(image string) string {
733 image = strings.TrimSpace(image)
734 if i := strings.LastIndex(image, "@"); i >= 0 {
735 image = image[:i]
736 }
737 slash := strings.LastIndex(image, "/")
738 name := image
739 if slash >= 0 {
740 name = image[slash+1:]
741 }
742 if i := strings.Index(name, ":"); i >= 0 {
743 name = name[:i]
744 }
745 return name
746}
747
748func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
749 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
750 if err != nil {
751 return err
752 }
753 wanted := map[string]bool{}
754 if job.NeedsSet {
755 for _, n := range job.Needs {
756 wanted[n] = true
757 }
758 } else {
759 stageOrder := map[string]int{}
760 n := 0
761 for _, j := range jobs {
762 if _, ok := stageOrder[j.Stage]; !ok {
763 stageOrder[j.Stage] = n
764 n++
765 }
766 }
767 my := stageOrder[job.Stage]
768 for _, j := range jobs {
769 if stageOrder[j.Stage] < my {
770 wanted[j.Name] = true
771 }
772 }
773 }
774 for _, j := range jobs {
775 if !wanted[j.Name] || j.Status != "success" {
776 continue
777 }
778 for _, rel := range j.ArtifactPaths {
779 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
780 if err != nil {
781 continue
782 }
783 dst := filepath.Join(work, filepath.FromSlash(rel))
784 if err := copyFile(src, dst); err != nil {
785 return err
786 }
787 }
788 }
789 return nil
790}
791
792func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
793 var expires *time.Time
794 if job.ArtifactExpire > 0 {
795 t := time.Now().Add(job.ArtifactExpire)
796 expires = &t
797 }
798 for _, rel := range job.ArtifactPaths {
799 src := filepath.Join(work, filepath.FromSlash(rel))
800 info, err := os.Stat(src)
801 if err != nil || info.IsDir() {
802 return fmt.Errorf("artifact %s ontbreekt", rel)
803 }
804 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
805 if err := copyFile(src, dst); err != nil {
806 return err
807 }
808 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
809 return err
810 }
811 }
812 return nil
813}
814
815func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
816 name, _ := ci.SplitRef(p.Ref)
817 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
818 if err != nil {
819 return nil, nil, err
820 }
821 refProtected := access.BranchProtected(patterns, name)
822 env := map[string]string{}
823 var mask []string
824 if repo.OwnerKind == "group" {
825 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
826 if err != nil {
827 return nil, nil, err
828 }
829 for k, v := range groupSecrets {
830 env[k] = v
831 mask = append(mask, v)
832 }
833 }
834 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
835 if err != nil {
836 return nil, nil, err
837 }
838 for k, v := range repoSecrets {
839 env[k] = v
840 mask = append(mask, v)
841 }
842 for k, v := range job.Variables {
843 env[k] = v
844 }
845 base := strings.TrimRight(r.BaseURL, "/")
846 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
847 var info *ci.MRInfo
848 if p.MRID != 0 {
849 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
850 target := ""
851 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
852 target = strings.TrimSpace(string(out))
853 }
854 info = &ci.MRInfo{
855 ID: m.ID, IID: m.Number, Title: m.Title,
856 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
857 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
858 TargetSHA: target,
859 }
860 }
861 }
862 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
863 pre := ci.Variables(ci.Ref{
864 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
865 DefaultBranch: repo.DefaultBranch, Source: p.Source,
866 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
867 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
868 })
869 for k, v := range pre {
870 env[k] = v
871 }
872 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
873 env["CI_JOB_NAME"] = job.Name
874 env["CI_JOB_STAGE"] = job.Stage
875 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
876 env["CI_JOB_TOKEN"] = token
877 if job.Environment != "" {
878 env["CI_ENVIRONMENT_NAME"] = job.Environment
879 if job.EnvironmentURL != "" {
880 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
881 }
882 }
883 for k, v := range job.Variables {
884 if strings.HasPrefix(k, "CI_NODE_") {
885 env[k] = v
886 }
887 }
888 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
889 // en safe.directory er alsnog bij komt.
890 markWorkspaceSafe(env)
891 JobGitAuth(env, base+"/"+repo.FullPath(), token)
892 mask = append(mask, token)
893 return env, mask, nil
894}
895
896// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
897// job-token. De header geldt alleen voor de URL van dit project (met en
898// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
899// het in elk image, zonder bestanden in de container.
900func JobGitAuth(env map[string]string, projectURL, token string) {
901 if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
902 return
903 }
904 n := 0
905 if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
906 n = v
907 }
908 for _, u := range []string{projectURL, projectURL + ".git"} {
909 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
910 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
911 n++
912 }
913 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
914}
915
916// markWorkspaceSafe zet safe.directory voor elke job.
917// De checkout is van de host-uid; de container-user is dat niet.
918// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
919// * of /workspace zet, blijft zoals hij is.
920func markWorkspaceSafe(env map[string]string) {
921 const key = "safe.directory"
922 n := 0
923 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
924 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
925 n = v
926 }
927 }
928 for i := 0; i < n; i++ {
929 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
930 continue
931 }
932 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
933 case "*", "/workspace":
934 return
935 }
936 }
937 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
938 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
939 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
940}
941
942func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
943 netName := fmt.Sprintf("bluegit-%d", job.ID)
944 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
945 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
946 }
947 var names []string
948 cleanup := func() {
949 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
950 defer cancel()
951 for _, name := range names {
952 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
953 }
954 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
955 }
956 for i, svc := range job.Services {
957 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
958 args := []string{"run", "-d", "--name", cname, "--network", netName}
959 if svc.Alias != "" {
960 args = append(args, "--network-alias", svc.Alias)
961 }
962 cmdEnv := append([]string{}, os.Environ()...)
963 for k, v := range svc.Variables {
964 args = append(args, "-e", k)
965 cmdEnv = append(cmdEnv, k+"="+v)
966 }
967 if len(svc.Entrypoint) > 0 {
968 args = append(args, "--entrypoint", svc.Entrypoint[0])
969 }
970 args = append(args, svc.Name)
971 if len(svc.Entrypoint) > 1 {
972 args = append(args, svc.Entrypoint[1:]...)
973 }
974 args = append(args, svc.Command...)
975 cmd := exec.CommandContext(ctx, r.Docker, args...)
976 cmd.Env = cmdEnv
977 var stderr bytes.Buffer
978 cmd.Stderr = &stderr
979 if err := cmd.Run(); err != nil {
980 cleanup()
981 msg := strings.TrimSpace(stderr.String())
982 if msg == "" {
983 msg = err.Error()
984 }
985 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
986 }
987 names = append(names, cname)
988 }
989 return netName, cleanup, nil
990}
991
992func shouldRetry(job store.Job, kind string) bool {
993 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
994 return false
995 }
996 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
997 return true
998 }
999 if containsStr(job.RetryWhen, kind) {
1000 return true
1001 }
1002 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
1003}
1004
1005func classifyFailure(err error) string {
1006 msg := err.Error()
1007 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
1008 return "runner_system_failure"
1009 }
1010 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
1011 return "job_execution_timeout"
1012 }
1013 return "script_failure"
1014}
1015
1016func containsStr(list []string, s string) bool {
1017 for _, item := range list {
1018 if item == s {
1019 return true
1020 }
1021 }
1022 return false
1023}
1024
1025func safeSegment(name string) string {
1026 var b strings.Builder
1027 for _, r := range name {
1028 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1029 b.WriteRune(r)
1030 } else {
1031 b.WriteByte('_')
1032 }
1033 }
1034 if b.Len() == 0 {
1035 return "job"
1036 }
1037 return b.String()
1038}
1039
1040func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
1041 if len(job.CachePaths) == 0 {
1042 return nil
1043 }
1044 root := r.cacheDir(repoID, job.CacheKey)
1045 for _, rel := range job.CachePaths {
1046 src := filepath.Join(root, filepath.FromSlash(rel))
1047 if _, err := os.Stat(src); err != nil {
1048 continue
1049 }
1050 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
1051 return err
1052 }
1053 }
1054 return nil
1055}
1056
1057func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1058 if len(job.CachePaths) == 0 {
1059 return nil
1060 }
1061 root := r.cacheDir(repoID, job.CacheKey)
1062 for _, rel := range job.CachePaths {
1063 src := filepath.Join(work, filepath.FromSlash(rel))
1064 info, err := os.Stat(src)
1065 if err != nil || info.IsDir() {
1066 continue
1067 }
1068 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1069 return err
1070 }
1071 }
1072 return nil
1073}
1074
1075func (r *Runner) cacheDir(repoID int64, key string) string {
1076 if key == "" {
1077 key = "default"
1078 }
1079 var b strings.Builder
1080 for _, r := range key {
1081 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1082 b.WriteRune(r)
1083 }
1084 }
1085 safe := b.String()
1086 if safe == "" {
1087 safe = "default"
1088 }
1089 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1090}
1091
1092func stamp(line string) string {
1093 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1094}
1095
1096type logSink struct {
1097 mu sync.Mutex
1098 store *store.Store
1099 jobID int64
1100 out func(string)
1101 secrets []string
1102 partial string
1103 current string
1104 opened time.Time
1105}
1106
1107func (l *logSink) Write(p []byte) (int, error) {
1108 l.mu.Lock()
1109 defer l.mu.Unlock()
1110 l.partial += string(p)
1111 for {
1112 i := strings.IndexByte(l.partial, '\n')
1113 if i < 0 {
1114 break
1115 }
1116 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1117 l.partial = l.partial[i+1:]
1118 }
1119 return len(p), nil
1120}
1121
1122func (l *logSink) flush() {
1123 l.mu.Lock()
1124 defer l.mu.Unlock()
1125 if strings.TrimSpace(l.partial) != "" {
1126 l.emit(l.partial)
1127 }
1128 l.partial = ""
1129}
1130
1131func (l *logSink) emit(line string) {
1132 line = ci.Mask(line, l.secrets)
1133 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1134 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1135 dur := int(time.Since(l.opened).Seconds())
1136 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1137 l.current = ""
1138 }
1139 if strings.HasPrefix(line, "section_start:") {
1140 l.current = line
1141 l.opened = time.Now()
1142 }
1143 }
1144 if l.out != nil {
1145 l.out(stamp(line))
1146 return
1147 }
1148 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1149}
1150
1151func (l *logSink) section(name, title string) {
1152 l.emit("section_start:" + name + ":" + title)
1153}
1154
1155func (l *logSink) closeSection() {
1156 if l.current == "" {
1157 return
1158 }
1159 name := "prepare"
1160 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1161 name = parts[0]
1162 }
1163 l.emit("section_end:" + name + ":done")
1164}
1165
1166func copyFile(src, dst string) error {
1167 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1168 return err
1169 }
1170 in, err := os.Open(src)
1171 if err != nil {
1172 return err
1173 }
1174 defer in.Close()
1175 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1176 if err != nil {
1177 return err
1178 }
1179 defer out.Close()
1180 _, err = io.Copy(out, in)
1181 return err
1182}
1183
1184func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1185 if bin == "" {
1186 bin = "git"
1187 }
1188 cmd := exec.CommandContext(ctx, bin, args...)
1189 var stdout, stderr bytes.Buffer
1190 cmd.Stdout = &stdout
1191 cmd.Stderr = &stderr
1192 if err := cmd.Run(); err != nil {
1193 msg := strings.TrimSpace(stderr.String())
1194 if msg == "" {
1195 msg = err.Error()
1196 }
1197 return "", fmt.Errorf("%s", msg)
1198 }
1199 return stdout.String(), nil
1200}
1201
1202// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1203// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1204// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1205func cloneArgs(bare, work string) []string {
1206 return []string{"clone", "--no-hardlinks", bare, work}
1207}