BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 29.4 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strconv"
15 "strings"
16 "sync"
17 "time"
18
19 "gitlab.com/Timfa/bluegit/internal/access"
20 "gitlab.com/Timfa/bluegit/internal/ci"
21 "gitlab.com/Timfa/bluegit/internal/gitrepo"
22 "gitlab.com/Timfa/bluegit/internal/images"
23 "gitlab.com/Timfa/bluegit/internal/store"
24 "gitlab.com/Timfa/bluegit/internal/webhook"
25)
26
27// Runner plant jobs en stuurt het resultaat door naar webhooks.
28type Runner struct {
29 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
30 Hold func(store.Pipeline) bool
31 Store *store.Store
32 GitBin string
33 Docker string
34 Storage string
35 BaseURL string
36 Network string
37 Name string
38 Timeout time.Duration
39 Log *slog.Logger
40 OnEvent func(repoID int64, ev webhook.Event)
41 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
42 Presets map[string]images.Override
43 slots chan struct{}
44 mu sync.RWMutex
45 cancelMu sync.Mutex
46 cancel map[int64]context.CancelFunc
47}
48
49// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
50func (r *Runner) Interrupt(ids []int64) {
51 r.cancelMu.Lock()
52 defer r.cancelMu.Unlock()
53 for _, id := range ids {
54 if c := r.cancel[id]; c != nil {
55 c()
56 }
57 }
58}
59
60func (r *Runner) track(id int64, cancel context.CancelFunc) {
61 r.cancelMu.Lock()
62 defer r.cancelMu.Unlock()
63 if r.cancel == nil {
64 r.cancel = map[int64]context.CancelFunc{}
65 }
66 r.cancel[id] = cancel
67}
68
69func (r *Runner) untrack(id int64) {
70 r.cancelMu.Lock()
71 defer r.cancelMu.Unlock()
72 delete(r.cancel, id)
73}
74
75// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
76func (r *Runner) SetStorage(path string) {
77 r.mu.Lock()
78 r.Storage = path
79 r.mu.Unlock()
80}
81
82func (r *Runner) storage() string {
83 r.mu.RLock()
84 defer r.mu.RUnlock()
85 return r.Storage
86}
87
88// Loop blijft jobs oppakken tot ctx afloopt.
89func (r *Runner) Loop(ctx context.Context, concurrency int) {
90 if concurrency < 1 {
91 concurrency = 1
92 }
93 r.slots = make(chan struct{}, concurrency)
94 tick := time.NewTicker(2 * time.Second)
95 defer tick.Stop()
96 for {
97 select {
98 case <-ctx.Done():
99 return
100 case <-tick.C:
101 r.Tick(ctx)
102 }
103 }
104}
105
106// Tick bekijkt alle open pipelines één keer.
107// Zonder opslagpad gebeurt er niets.
108func (r *Runner) Tick(ctx context.Context) {
109 if r.storage() == "" {
110 return
111 }
112 pipes, err := r.Store.ActivePipelines(ctx)
113 if err != nil {
114 r.Log.Error("pipelines", "err", err)
115 return
116 }
117 for _, p := range pipes {
118 r.schedule(ctx, p)
119 }
120}
121
122func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
123 r.settle(ctx, p.ID)
124 if r.Hold != nil && r.Hold(p) {
125 return // wacht op de self-update
126 }
127 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
128 if err != nil {
129 r.Log.Error("jobs", "err", err)
130 return
131 }
132 specs := make([]ci.Job, len(jobs))
133 status := map[string]string{}
134 for i, j := range jobs {
135 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
136 status[j.Name] = j.Status
137 }
138 for _, j := range jobs {
139 if j.Status != "queued" && j.Status != "manual" {
140 continue
141 }
142 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
143 case "skip":
144 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
145 status[j.Name] = "skipped"
146 case "fail":
147 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
148 status[j.Name] = "failed"
149 case "ready":
150 if j.Status != "queued" {
151 continue
152 }
153 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
154 continue
155 }
156 // Meer jobs dan ci.concurrency blijven queued, met een plek
157 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
158 if !r.acquireSlot() {
159 continue
160 }
161 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
162 if err != nil || !ok {
163 r.releaseSlot()
164 continue
165 }
166 status[j.Name] = "running"
167 job := j
168 pipe := p
169 go r.execute(ctx, pipe, job)
170 }
171 }
172}
173
174// settle slaat jobs over die door een harde mislukking of een
175// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
176// pipelinestatus al op failed was gezet.
177func (r *Runner) settle(ctx context.Context, pipelineID int64) {
178 for {
179 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
180 if err != nil {
181 if r.Log != nil {
182 r.Log.Error("jobs", "err", err)
183 }
184 return
185 }
186 specs := make([]ci.Job, len(jobs))
187 status := map[string]string{}
188 for i, j := range jobs {
189 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
190 status[j.Name] = j.Status
191 }
192 moved := false
193 for _, j := range jobs {
194 if j.Status != "queued" && j.Status != "manual" {
195 continue
196 }
197 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
198 case "skip":
199 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
200 r.Log.Error("skip", "err", err)
201 continue
202 }
203 status[j.Name] = "skipped"
204 moved = true
205 case "fail":
206 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
207 r.Log.Error("fail", "err", err)
208 continue
209 }
210 status[j.Name] = "failed"
211 moved = true
212 }
213 }
214 if !moved {
215 return
216 }
217 }
218}
219
220func indexOf(jobs []store.Job, name string) int {
221 for i, j := range jobs {
222 if j.Name == name {
223 return i
224 }
225 }
226 return 0
227}
228
229func (r *Runner) acquireSlot() bool {
230 if r.slots == nil {
231 return true
232 }
233 select {
234 case r.slots <- struct{}{}:
235 return true
236 default:
237 return false
238 }
239}
240
241func (r *Runner) releaseSlot() {
242 if r.slots == nil {
243 return
244 }
245 select {
246 case <-r.slots:
247 default:
248 }
249}
250
251func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
252 defer r.releaseSlot()
253 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
254 base, stop := context.WithCancel(ctx)
255 r.track(job.ID, stop)
256 defer func() {
257 r.untrack(job.ID)
258 stop()
259 }()
260 runCtx := base
261 limit := r.Timeout
262 if job.Timeout > 0 {
263 limit = job.Timeout
264 }
265 if limit > 0 {
266 var cancel context.CancelFunc
267 runCtx, cancel = context.WithTimeout(base, limit)
268 defer cancel()
269 }
270 mask, err := r.runOne(runCtx, p, job)
271 status := "success"
272 extra := ""
273 kind := ""
274 switch {
275 case err == nil:
276 case ctx.Err() == nil && base.Err() != nil:
277 status = "canceled"
278 extra = stamp("canceled because a newer pipeline started for the same ref")
279 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
280 status = "failed"
281 kind = "job_execution_timeout"
282 extra = stamp(ci.Mask(err.Error(), mask))
283 default:
284 status = "failed"
285 kind = classifyFailure(err)
286 extra = stamp(ci.Mask(err.Error(), mask))
287 }
288 if status == "failed" && shouldRetry(job, kind) {
289 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
290 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
291 r.Log.Error("job opnieuw", "err", ferr)
292 } else if ferr == nil {
293 r.settle(context.Background(), p.ID)
294 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
295 r.Log.Error("pipelinestatus", "err", rerr)
296 }
297 return
298 }
299 }
300 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
301 r.Log.Error("job afronden", "err", ferr)
302 }
303 r.settle(context.Background(), p.ID)
304 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
305 if rerr != nil {
306 r.Log.Error("pipelinestatus", "err", rerr)
307 return
308 }
309 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
310 kind := webhook.EventPipelineSuccess
311 word := "geslaagd"
312 if final == "failed" {
313 kind = webhook.EventPipelineFailure
314 word = "mislukt"
315 }
316 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
317 if err != nil {
318 return
319 }
320 branch, _ := gitrepo.BranchOfRef(p.Ref)
321 if branch == "" {
322 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
323 }
324 r.OnEvent(p.RepoID, webhook.Event{
325 Type: kind,
326 Project: repo.FullPath(),
327 Branch: branch,
328 Commit: p.CommitSHA,
329 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
330 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
331 PipelineID: fmt.Sprint(p.ID),
332 })
333 }
334}
335
336func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
337 repo, err := r.Store.RepoByID(ctx, p.RepoID)
338 if err != nil {
339 return nil, err
340 }
341 sink := &logSink{store: r.Store, jobID: job.ID}
342 sink.section("prepare", "Preparing the docker executor")
343 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
344 _ = os.RemoveAll(work)
345 if err := os.MkdirAll(work, 0o755); err != nil {
346 sink.closeSection()
347 return nil, err
348 }
349 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
350 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
351 sink.closeSection()
352 return nil, fmt.Errorf("clone: %w", err)
353 }
354 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
355 sink.closeSection()
356 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
357 }
358 if err := r.restoreCache(repo.ID, job, work); err != nil {
359 sink.closeSection()
360 return nil, err
361 }
362 if err := r.materialize(ctx, p, job, work); err != nil {
363 sink.closeSection()
364 return nil, err
365 }
366 scriptPath := filepath.Join(work, ".bluegit-job.sh")
367 host, _ := os.Hostname()
368 script := shellScript(job, placeNote{
369 Runner: r.Name, Host: host, WorkDir: "/workspace",
370 PipelineID: p.ID, JobID: job.ID,
371 })
372 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
373 sink.closeSection()
374 return nil, err
375 }
376 token, err := r.Store.CreateJobToken(ctx, job.ID)
377 if err != nil {
378 sink.closeSection()
379 return nil, err
380 }
381 env, mask, err := r.environment(ctx, repo, p, job, token)
382 if err != nil {
383 sink.closeSection()
384 return nil, err
385 }
386 sink.secrets = mask
387 ref, lines, rerr := r.resolveImage(ctx, job.Image)
388 for _, line := range lines {
389 sink.emit(line)
390 }
391 if rerr != nil {
392 sink.closeSection()
393 return mask, rerr
394 }
395 job.Image = ref
396 for i := range job.Services {
397 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
398 for _, line := range slines {
399 sink.emit(line)
400 }
401 if serr != nil {
402 sink.closeSection()
403 return mask, serr
404 }
405 job.Services[i].Name = sref
406 }
407 network := r.Network
408 if len(job.Services) > 0 {
409 net, cleanup, serr := r.startServices(ctx, job)
410 if serr != nil {
411 sink.closeSection()
412 return mask, serr
413 }
414 defer cleanup()
415 network = net
416 }
417 sink.closeSection()
418 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
419 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
420 // staat de waarde niet in de proceslijst.
421 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
422 cmdEnv := append([]string{}, os.Environ()...)
423 for k, v := range env {
424 args = append(args, "-e", k)
425 cmdEnv = append(cmdEnv, k+"="+v)
426 }
427 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
428 cmd := exec.CommandContext(ctx, r.Docker, args...)
429 cmd.Env = cmdEnv
430 cmd.Stdout = sink
431 cmd.Stderr = sink
432 err = cmd.Run()
433 sink.flush()
434 if err != nil {
435 return mask, fmt.Errorf("container ended with an error: %w", err)
436 }
437 if err := r.collect(ctx, job, work); err != nil {
438 return mask, err
439 }
440 if err := r.saveCache(repo.ID, job, work); err != nil {
441 return mask, err
442 }
443 return mask, nil
444}
445
446// placeNote zijn de vaste regels in de sectie Preparing environment.
447type placeNote struct {
448 Runner string
449 Host string
450 WorkDir string
451 PipelineID int64
452 JobID int64
453}
454
455func (n placeNote) lines() []string {
456 runner := strings.TrimSpace(n.Runner)
457 if runner == "" {
458 runner = "Steve"
459 }
460 host := strings.TrimSpace(n.Host)
461 if host == "" {
462 host = "localhost"
463 }
464 dir := strings.TrimSpace(n.WorkDir)
465 if dir == "" {
466 dir = "/workspace"
467 }
468 return []string{
469 fmt.Sprintf("Running on %s via %s...", runner, host),
470 fmt.Sprintf("Working directory: %s", dir),
471 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
472 }
473}
474
475func shSingle(s string) string {
476 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
477}
478
479func shellScript(job store.Job, note placeNote) string {
480 var b strings.Builder
481 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
482 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
483 for _, line := range note.lines() {
484 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
485 }
486 if strings.TrimSpace(job.Before) != "" {
487 b.WriteString("(\nset -e\n")
488 b.WriteString(job.Before)
489 if !strings.HasSuffix(job.Before, "\n") {
490 b.WriteByte('\n')
491 }
492 b.WriteString(")\nbrc=$?\n")
493 } else {
494 b.WriteString("brc=0\n")
495 }
496 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
497 b.WriteString("echo \"section_start:script:Running script\"\n")
498 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
499 b.WriteString(job.Script)
500 if !strings.HasSuffix(job.Script, "\n") {
501 b.WriteByte('\n')
502 }
503 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
504 b.WriteString("echo \"section_end:script:Running script\"\n")
505 b.WriteString("echo \"section_start:after:After script\"\n")
506 if strings.TrimSpace(job.After) != "" {
507 b.WriteString("(\nset -e\n")
508 b.WriteString(job.After)
509 if !strings.HasSuffix(job.After, "\n") {
510 b.WriteByte('\n')
511 }
512 b.WriteString(")\n")
513 }
514 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
515 return b.String()
516}
517
518// resolveImage kiest het image waarmee de container start.
519// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
520func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
521 image = strings.TrimSpace(image)
522 set, err := images.Effective(r.Presets)
523 if err != nil {
524 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
525 }
526 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
527 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
528 if p, ok := images.Find(set, image); ok {
529 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
530 }
531 return image, dockerImageLines(image, "", nil), missing
532 }
533 p, ok := images.Find(set, image)
534 if !ok {
535 id, digests := r.inspectImage(ctx, image)
536 return image, dockerImageLines(image, id, digests), nil
537 }
538 ref, built, err := p.Ensure(ctx, r.Docker)
539 if err != nil {
540 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
541 }
542 var lines []string
543 if built {
544 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
545 }
546 id, digests := r.inspectImage(ctx, ref)
547 var baseDigests []string
548 if ref != p.From {
549 _, baseDigests = r.inspectImage(ctx, p.From)
550 }
551 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
552 return ref, lines, nil
553}
554
555func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
556 var lines []string
557 if preset != "" {
558 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
559 }
560 if ref == "" {
561 return lines
562 }
563 lines = append(lines, dockerImageLines(ref, id, digests)...)
564 if preset != "" && ref != base {
565 digest := matchingDigest(base, baseDigests)
566 if digest == "" && len(baseDigests) > 0 {
567 digest = strings.TrimSpace(baseDigests[0])
568 }
569 if digest != "" {
570 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
571 } else {
572 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
573 }
574 }
575 return lines
576}
577
578func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
579 if strings.TrimSpace(image) == "" || r.Docker == "" {
580 return "", nil
581 }
582 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
583 if err != nil {
584 return "", nil
585 }
586 return parseImageInspect(out)
587}
588
589func parseImageInspect(out string) (string, []string) {
590 lines := strings.Split(out, "\n")
591 id := ""
592 if len(lines) > 0 {
593 id = strings.TrimSpace(lines[0])
594 }
595 var digests []string
596 for _, line := range lines[1:] {
597 line = strings.TrimSpace(line)
598 if line != "" {
599 digests = append(digests, line)
600 }
601 }
602 return id, digests
603}
604
605func dockerImageLines(image, id string, digests []string) []string {
606 image = strings.TrimSpace(image)
607 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
608 id = strings.TrimSpace(id)
609 digest := matchingDigest(image, digests)
610 switch {
611 case id != "" && digest != "":
612 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
613 case digest != "":
614 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
615 case id != "":
616 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
617 }
618 return lines
619}
620
621func matchingDigest(image string, digests []string) string {
622 name := imageName(image)
623 fallback := ""
624 for _, digest := range digests {
625 digest = strings.TrimSpace(digest)
626 at := strings.LastIndex(digest, "@")
627 if digest == "" || at <= 0 {
628 continue
629 }
630 if fallback == "" {
631 fallback = digest
632 }
633 repo := digest[:at]
634 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
635 return digest
636 }
637 }
638 return fallback
639}
640
641func imageName(image string) string {
642 image = strings.TrimSpace(image)
643 if i := strings.LastIndex(image, "@"); i >= 0 {
644 image = image[:i]
645 }
646 slash := strings.LastIndex(image, "/")
647 name := image
648 if slash >= 0 {
649 name = image[slash+1:]
650 }
651 if i := strings.Index(name, ":"); i >= 0 {
652 name = name[:i]
653 }
654 return name
655}
656
657func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
658 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
659 if err != nil {
660 return err
661 }
662 wanted := map[string]bool{}
663 if job.NeedsSet {
664 for _, n := range job.Needs {
665 wanted[n] = true
666 }
667 } else {
668 stageOrder := map[string]int{}
669 n := 0
670 for _, j := range jobs {
671 if _, ok := stageOrder[j.Stage]; !ok {
672 stageOrder[j.Stage] = n
673 n++
674 }
675 }
676 my := stageOrder[job.Stage]
677 for _, j := range jobs {
678 if stageOrder[j.Stage] < my {
679 wanted[j.Name] = true
680 }
681 }
682 }
683 for _, j := range jobs {
684 if !wanted[j.Name] || j.Status != "success" {
685 continue
686 }
687 for _, rel := range j.ArtifactPaths {
688 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
689 if err != nil {
690 continue
691 }
692 dst := filepath.Join(work, filepath.FromSlash(rel))
693 if err := copyFile(src, dst); err != nil {
694 return err
695 }
696 }
697 }
698 return nil
699}
700
701func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
702 var expires *time.Time
703 if job.ArtifactExpire > 0 {
704 t := time.Now().Add(job.ArtifactExpire)
705 expires = &t
706 }
707 for _, rel := range job.ArtifactPaths {
708 src := filepath.Join(work, filepath.FromSlash(rel))
709 info, err := os.Stat(src)
710 if err != nil || info.IsDir() {
711 return fmt.Errorf("artifact %s ontbreekt", rel)
712 }
713 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
714 if err := copyFile(src, dst); err != nil {
715 return err
716 }
717 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
718 return err
719 }
720 }
721 return nil
722}
723
724func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
725 name, _ := ci.SplitRef(p.Ref)
726 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
727 if err != nil {
728 return nil, nil, err
729 }
730 refProtected := access.BranchProtected(patterns, name)
731 env := map[string]string{}
732 var mask []string
733 if repo.OwnerKind == "group" {
734 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
735 if err != nil {
736 return nil, nil, err
737 }
738 for k, v := range groupSecrets {
739 env[k] = v
740 mask = append(mask, v)
741 }
742 }
743 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
744 if err != nil {
745 return nil, nil, err
746 }
747 for k, v := range repoSecrets {
748 env[k] = v
749 mask = append(mask, v)
750 }
751 for k, v := range job.Variables {
752 env[k] = v
753 }
754 base := strings.TrimRight(r.BaseURL, "/")
755 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
756 var info *ci.MRInfo
757 if p.MRID != 0 {
758 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
759 target := ""
760 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
761 target = strings.TrimSpace(string(out))
762 }
763 info = &ci.MRInfo{
764 ID: m.ID, IID: m.Number, Title: m.Title,
765 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
766 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
767 TargetSHA: target,
768 }
769 }
770 }
771 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
772 pre := ci.Variables(ci.Ref{
773 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
774 DefaultBranch: repo.DefaultBranch, Source: p.Source,
775 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
776 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
777 })
778 for k, v := range pre {
779 env[k] = v
780 }
781 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
782 env["CI_JOB_NAME"] = job.Name
783 env["CI_JOB_STAGE"] = job.Stage
784 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
785 env["CI_JOB_TOKEN"] = token
786 if job.Environment != "" {
787 env["CI_ENVIRONMENT_NAME"] = job.Environment
788 if job.EnvironmentURL != "" {
789 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
790 }
791 }
792 for k, v := range job.Variables {
793 if strings.HasPrefix(k, "CI_NODE_") {
794 env[k] = v
795 }
796 }
797 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
798 // en safe.directory er alsnog bij komt.
799 markWorkspaceSafe(env)
800 mask = append(mask, token)
801 return env, mask, nil
802}
803
804// markWorkspaceSafe zet safe.directory voor elke job.
805// De checkout is van de host-uid; de container-user is dat niet.
806// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
807// * of /workspace zet, blijft zoals hij is.
808func markWorkspaceSafe(env map[string]string) {
809 const key = "safe.directory"
810 n := 0
811 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
812 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
813 n = v
814 }
815 }
816 for i := 0; i < n; i++ {
817 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
818 continue
819 }
820 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
821 case "*", "/workspace":
822 return
823 }
824 }
825 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
826 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
827 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
828}
829
830func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
831 netName := fmt.Sprintf("bluegit-%d", job.ID)
832 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
833 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
834 }
835 var names []string
836 cleanup := func() {
837 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
838 defer cancel()
839 for _, name := range names {
840 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
841 }
842 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
843 }
844 for i, svc := range job.Services {
845 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
846 args := []string{"run", "-d", "--name", cname, "--network", netName}
847 if svc.Alias != "" {
848 args = append(args, "--network-alias", svc.Alias)
849 }
850 cmdEnv := append([]string{}, os.Environ()...)
851 for k, v := range svc.Variables {
852 args = append(args, "-e", k)
853 cmdEnv = append(cmdEnv, k+"="+v)
854 }
855 if len(svc.Entrypoint) > 0 {
856 args = append(args, "--entrypoint", svc.Entrypoint[0])
857 }
858 args = append(args, svc.Name)
859 if len(svc.Entrypoint) > 1 {
860 args = append(args, svc.Entrypoint[1:]...)
861 }
862 args = append(args, svc.Command...)
863 cmd := exec.CommandContext(ctx, r.Docker, args...)
864 cmd.Env = cmdEnv
865 var stderr bytes.Buffer
866 cmd.Stderr = &stderr
867 if err := cmd.Run(); err != nil {
868 cleanup()
869 msg := strings.TrimSpace(stderr.String())
870 if msg == "" {
871 msg = err.Error()
872 }
873 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
874 }
875 names = append(names, cname)
876 }
877 return netName, cleanup, nil
878}
879
880func shouldRetry(job store.Job, kind string) bool {
881 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
882 return false
883 }
884 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
885 return true
886 }
887 if containsStr(job.RetryWhen, kind) {
888 return true
889 }
890 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
891}
892
893func classifyFailure(err error) string {
894 msg := err.Error()
895 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
896 return "runner_system_failure"
897 }
898 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
899 return "job_execution_timeout"
900 }
901 return "script_failure"
902}
903
904func containsStr(list []string, s string) bool {
905 for _, item := range list {
906 if item == s {
907 return true
908 }
909 }
910 return false
911}
912
913func safeSegment(name string) string {
914 var b strings.Builder
915 for _, r := range name {
916 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
917 b.WriteRune(r)
918 } else {
919 b.WriteByte('_')
920 }
921 }
922 if b.Len() == 0 {
923 return "job"
924 }
925 return b.String()
926}
927
928func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
929 if len(job.CachePaths) == 0 {
930 return nil
931 }
932 root := r.cacheDir(repoID, job.CacheKey)
933 for _, rel := range job.CachePaths {
934 src := filepath.Join(root, filepath.FromSlash(rel))
935 if _, err := os.Stat(src); err != nil {
936 continue
937 }
938 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
939 return err
940 }
941 }
942 return nil
943}
944
945func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
946 if len(job.CachePaths) == 0 {
947 return nil
948 }
949 root := r.cacheDir(repoID, job.CacheKey)
950 for _, rel := range job.CachePaths {
951 src := filepath.Join(work, filepath.FromSlash(rel))
952 info, err := os.Stat(src)
953 if err != nil || info.IsDir() {
954 continue
955 }
956 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
957 return err
958 }
959 }
960 return nil
961}
962
963func (r *Runner) cacheDir(repoID int64, key string) string {
964 if key == "" {
965 key = "default"
966 }
967 var b strings.Builder
968 for _, r := range key {
969 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
970 b.WriteRune(r)
971 }
972 }
973 safe := b.String()
974 if safe == "" {
975 safe = "default"
976 }
977 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
978}
979
980func stamp(line string) string {
981 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
982}
983
984type logSink struct {
985 mu sync.Mutex
986 store *store.Store
987 jobID int64
988 secrets []string
989 partial string
990 current string
991 opened time.Time
992}
993
994func (l *logSink) Write(p []byte) (int, error) {
995 l.mu.Lock()
996 defer l.mu.Unlock()
997 l.partial += string(p)
998 for {
999 i := strings.IndexByte(l.partial, '\n')
1000 if i < 0 {
1001 break
1002 }
1003 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1004 l.partial = l.partial[i+1:]
1005 }
1006 return len(p), nil
1007}
1008
1009func (l *logSink) flush() {
1010 l.mu.Lock()
1011 defer l.mu.Unlock()
1012 if strings.TrimSpace(l.partial) != "" {
1013 l.emit(l.partial)
1014 }
1015 l.partial = ""
1016}
1017
1018func (l *logSink) emit(line string) {
1019 line = ci.Mask(line, l.secrets)
1020 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1021 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1022 dur := int(time.Since(l.opened).Seconds())
1023 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1024 l.current = ""
1025 }
1026 if strings.HasPrefix(line, "section_start:") {
1027 l.current = line
1028 l.opened = time.Now()
1029 }
1030 }
1031 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1032}
1033
1034func (l *logSink) section(name, title string) {
1035 l.emit("section_start:" + name + ":" + title)
1036}
1037
1038func (l *logSink) closeSection() {
1039 if l.current == "" {
1040 return
1041 }
1042 name := "prepare"
1043 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1044 name = parts[0]
1045 }
1046 l.emit("section_end:" + name + ":done")
1047}
1048
1049func copyFile(src, dst string) error {
1050 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1051 return err
1052 }
1053 in, err := os.Open(src)
1054 if err != nil {
1055 return err
1056 }
1057 defer in.Close()
1058 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1059 if err != nil {
1060 return err
1061 }
1062 defer out.Close()
1063 _, err = io.Copy(out, in)
1064 return err
1065}
1066
1067func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1068 if bin == "" {
1069 bin = "git"
1070 }
1071 cmd := exec.CommandContext(ctx, bin, args...)
1072 var stdout, stderr bytes.Buffer
1073 cmd.Stdout = &stdout
1074 cmd.Stderr = &stderr
1075 if err := cmd.Run(); err != nil {
1076 msg := strings.TrimSpace(stderr.String())
1077 if msg == "" {
1078 msg = err.Error()
1079 }
1080 return "", fmt.Errorf("%s", msg)
1081 }
1082 return stdout.String(), nil
1083}
1084
1085// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1086// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1087// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1088func cloneArgs(bare, work string) []string {
1089 return []string{"clone", "--no-hardlinks", bare, work}
1090}