BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

fix: accepteer een checkout van een andere uid en toon het update-log
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
71de6a16 History
runner.go 28.9 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strconv"
15 "strings"
16 "sync"
17 "time"
18
19 "gitlab.com/Timfa/bluegit/internal/access"
20 "gitlab.com/Timfa/bluegit/internal/ci"
21 "gitlab.com/Timfa/bluegit/internal/gitrepo"
22 "gitlab.com/Timfa/bluegit/internal/images"
23 "gitlab.com/Timfa/bluegit/internal/store"
24 "gitlab.com/Timfa/bluegit/internal/webhook"
25)
26
27// Runner plant jobs en stuurt het resultaat door naar webhooks.
28type Runner struct {
29 Store *store.Store
30 GitBin string
31 Docker string
32 Storage string
33 BaseURL string
34 Network string
35 Name string
36 Timeout time.Duration
37 Log *slog.Logger
38 OnEvent func(repoID int64, ev webhook.Event)
39 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
40 Presets map[string]images.Override
41 slots chan struct{}
42 mu sync.RWMutex
43 cancelMu sync.Mutex
44 cancel map[int64]context.CancelFunc
45}
46
47// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
48func (r *Runner) Interrupt(ids []int64) {
49 r.cancelMu.Lock()
50 defer r.cancelMu.Unlock()
51 for _, id := range ids {
52 if c := r.cancel[id]; c != nil {
53 c()
54 }
55 }
56}
57
58func (r *Runner) track(id int64, cancel context.CancelFunc) {
59 r.cancelMu.Lock()
60 defer r.cancelMu.Unlock()
61 if r.cancel == nil {
62 r.cancel = map[int64]context.CancelFunc{}
63 }
64 r.cancel[id] = cancel
65}
66
67func (r *Runner) untrack(id int64) {
68 r.cancelMu.Lock()
69 defer r.cancelMu.Unlock()
70 delete(r.cancel, id)
71}
72
73// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
74func (r *Runner) SetStorage(path string) {
75 r.mu.Lock()
76 r.Storage = path
77 r.mu.Unlock()
78}
79
80func (r *Runner) storage() string {
81 r.mu.RLock()
82 defer r.mu.RUnlock()
83 return r.Storage
84}
85
86// Loop blijft jobs oppakken tot ctx afloopt.
87func (r *Runner) Loop(ctx context.Context, concurrency int) {
88 if concurrency < 1 {
89 concurrency = 1
90 }
91 r.slots = make(chan struct{}, concurrency)
92 tick := time.NewTicker(2 * time.Second)
93 defer tick.Stop()
94 for {
95 select {
96 case <-ctx.Done():
97 return
98 case <-tick.C:
99 r.Tick(ctx)
100 }
101 }
102}
103
104// Tick bekijkt alle open pipelines één keer.
105// Zonder opslagpad gebeurt er niets.
106func (r *Runner) Tick(ctx context.Context) {
107 if r.storage() == "" {
108 return
109 }
110 pipes, err := r.Store.ActivePipelines(ctx)
111 if err != nil {
112 r.Log.Error("pipelines", "err", err)
113 return
114 }
115 for _, p := range pipes {
116 r.schedule(ctx, p)
117 }
118}
119
120func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
121 r.settle(ctx, p.ID)
122 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
123 if err != nil {
124 r.Log.Error("jobs", "err", err)
125 return
126 }
127 specs := make([]ci.Job, len(jobs))
128 status := map[string]string{}
129 for i, j := range jobs {
130 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
131 status[j.Name] = j.Status
132 }
133 for _, j := range jobs {
134 if j.Status != "queued" && j.Status != "manual" {
135 continue
136 }
137 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
138 case "skip":
139 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
140 status[j.Name] = "skipped"
141 case "fail":
142 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
143 status[j.Name] = "failed"
144 case "ready":
145 if j.Status != "queued" {
146 continue
147 }
148 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
149 continue
150 }
151 // Meer jobs dan ci.concurrency blijven queued, met een plek
152 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
153 if !r.acquireSlot() {
154 continue
155 }
156 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
157 if err != nil || !ok {
158 r.releaseSlot()
159 continue
160 }
161 status[j.Name] = "running"
162 job := j
163 pipe := p
164 go r.execute(ctx, pipe, job)
165 }
166 }
167}
168
169// settle slaat jobs over die door een harde mislukking of een
170// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
171// pipelinestatus al op failed was gezet.
172func (r *Runner) settle(ctx context.Context, pipelineID int64) {
173 for {
174 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
175 if err != nil {
176 if r.Log != nil {
177 r.Log.Error("jobs", "err", err)
178 }
179 return
180 }
181 specs := make([]ci.Job, len(jobs))
182 status := map[string]string{}
183 for i, j := range jobs {
184 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
185 status[j.Name] = j.Status
186 }
187 moved := false
188 for _, j := range jobs {
189 if j.Status != "queued" && j.Status != "manual" {
190 continue
191 }
192 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
193 case "skip":
194 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
195 r.Log.Error("skip", "err", err)
196 continue
197 }
198 status[j.Name] = "skipped"
199 moved = true
200 case "fail":
201 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
202 r.Log.Error("fail", "err", err)
203 continue
204 }
205 status[j.Name] = "failed"
206 moved = true
207 }
208 }
209 if !moved {
210 return
211 }
212 }
213}
214
215func indexOf(jobs []store.Job, name string) int {
216 for i, j := range jobs {
217 if j.Name == name {
218 return i
219 }
220 }
221 return 0
222}
223
224func (r *Runner) acquireSlot() bool {
225 if r.slots == nil {
226 return true
227 }
228 select {
229 case r.slots <- struct{}{}:
230 return true
231 default:
232 return false
233 }
234}
235
236func (r *Runner) releaseSlot() {
237 if r.slots == nil {
238 return
239 }
240 select {
241 case <-r.slots:
242 default:
243 }
244}
245
246func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
247 defer r.releaseSlot()
248 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
249 base, stop := context.WithCancel(ctx)
250 r.track(job.ID, stop)
251 defer func() {
252 r.untrack(job.ID)
253 stop()
254 }()
255 runCtx := base
256 limit := r.Timeout
257 if job.Timeout > 0 {
258 limit = job.Timeout
259 }
260 if limit > 0 {
261 var cancel context.CancelFunc
262 runCtx, cancel = context.WithTimeout(base, limit)
263 defer cancel()
264 }
265 mask, err := r.runOne(runCtx, p, job)
266 status := "success"
267 extra := ""
268 kind := ""
269 switch {
270 case err == nil:
271 case ctx.Err() == nil && base.Err() != nil:
272 status = "canceled"
273 extra = stamp("canceled because a newer pipeline started for the same ref")
274 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
275 status = "failed"
276 kind = "job_execution_timeout"
277 extra = stamp(ci.Mask(err.Error(), mask))
278 default:
279 status = "failed"
280 kind = classifyFailure(err)
281 extra = stamp(ci.Mask(err.Error(), mask))
282 }
283 if status == "failed" && shouldRetry(job, kind) {
284 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
285 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
286 r.Log.Error("job opnieuw", "err", ferr)
287 } else if ferr == nil {
288 r.settle(context.Background(), p.ID)
289 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
290 r.Log.Error("pipelinestatus", "err", rerr)
291 }
292 return
293 }
294 }
295 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
296 r.Log.Error("job afronden", "err", ferr)
297 }
298 r.settle(context.Background(), p.ID)
299 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
300 if rerr != nil {
301 r.Log.Error("pipelinestatus", "err", rerr)
302 return
303 }
304 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
305 kind := webhook.EventPipelineSuccess
306 word := "geslaagd"
307 if final == "failed" {
308 kind = webhook.EventPipelineFailure
309 word = "mislukt"
310 }
311 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
312 if err != nil {
313 return
314 }
315 branch, _ := gitrepo.BranchOfRef(p.Ref)
316 if branch == "" {
317 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
318 }
319 r.OnEvent(p.RepoID, webhook.Event{
320 Type: kind,
321 Project: repo.FullPath(),
322 Branch: branch,
323 Commit: p.CommitSHA,
324 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
325 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
326 PipelineID: fmt.Sprint(p.ID),
327 })
328 }
329}
330
331func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
332 repo, err := r.Store.RepoByID(ctx, p.RepoID)
333 if err != nil {
334 return nil, err
335 }
336 sink := &logSink{store: r.Store, jobID: job.ID}
337 sink.section("prepare", "Preparing the docker executor")
338 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
339 _ = os.RemoveAll(work)
340 if err := os.MkdirAll(work, 0o755); err != nil {
341 sink.closeSection()
342 return nil, err
343 }
344 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
345 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
346 sink.closeSection()
347 return nil, fmt.Errorf("clone: %w", err)
348 }
349 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
350 sink.closeSection()
351 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
352 }
353 if err := r.restoreCache(repo.ID, job, work); err != nil {
354 sink.closeSection()
355 return nil, err
356 }
357 if err := r.materialize(ctx, p, job, work); err != nil {
358 sink.closeSection()
359 return nil, err
360 }
361 scriptPath := filepath.Join(work, ".bluegit-job.sh")
362 host, _ := os.Hostname()
363 script := shellScript(job, placeNote{
364 Runner: r.Name, Host: host, WorkDir: "/workspace",
365 PipelineID: p.ID, JobID: job.ID,
366 })
367 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
368 sink.closeSection()
369 return nil, err
370 }
371 token, err := r.Store.CreateJobToken(ctx, job.ID)
372 if err != nil {
373 sink.closeSection()
374 return nil, err
375 }
376 env, mask, err := r.environment(ctx, repo, p, job, token)
377 if err != nil {
378 sink.closeSection()
379 return nil, err
380 }
381 sink.secrets = mask
382 ref, lines, rerr := r.resolveImage(ctx, job.Image)
383 for _, line := range lines {
384 sink.emit(line)
385 }
386 if rerr != nil {
387 sink.closeSection()
388 return mask, rerr
389 }
390 job.Image = ref
391 for i := range job.Services {
392 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
393 for _, line := range slines {
394 sink.emit(line)
395 }
396 if serr != nil {
397 sink.closeSection()
398 return mask, serr
399 }
400 job.Services[i].Name = sref
401 }
402 network := r.Network
403 if len(job.Services) > 0 {
404 net, cleanup, serr := r.startServices(ctx, job)
405 if serr != nil {
406 sink.closeSection()
407 return mask, serr
408 }
409 defer cleanup()
410 network = net
411 }
412 sink.closeSection()
413 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
414 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
415 // staat de waarde niet in de proceslijst.
416 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
417 cmdEnv := append([]string{}, os.Environ()...)
418 for k, v := range env {
419 args = append(args, "-e", k)
420 cmdEnv = append(cmdEnv, k+"="+v)
421 }
422 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
423 cmd := exec.CommandContext(ctx, r.Docker, args...)
424 cmd.Env = cmdEnv
425 cmd.Stdout = sink
426 cmd.Stderr = sink
427 err = cmd.Run()
428 sink.flush()
429 if err != nil {
430 return mask, fmt.Errorf("container ended with an error: %w", err)
431 }
432 if err := r.collect(ctx, job, work); err != nil {
433 return mask, err
434 }
435 if err := r.saveCache(repo.ID, job, work); err != nil {
436 return mask, err
437 }
438 return mask, nil
439}
440
441// placeNote zijn de vaste regels in de sectie Preparing environment.
442type placeNote struct {
443 Runner string
444 Host string
445 WorkDir string
446 PipelineID int64
447 JobID int64
448}
449
450func (n placeNote) lines() []string {
451 runner := strings.TrimSpace(n.Runner)
452 if runner == "" {
453 runner = "Steve"
454 }
455 host := strings.TrimSpace(n.Host)
456 if host == "" {
457 host = "localhost"
458 }
459 dir := strings.TrimSpace(n.WorkDir)
460 if dir == "" {
461 dir = "/workspace"
462 }
463 return []string{
464 fmt.Sprintf("Running on %s via %s...", runner, host),
465 fmt.Sprintf("Working directory: %s", dir),
466 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
467 }
468}
469
470func shSingle(s string) string {
471 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
472}
473
474func shellScript(job store.Job, note placeNote) string {
475 var b strings.Builder
476 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
477 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
478 for _, line := range note.lines() {
479 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
480 }
481 if strings.TrimSpace(job.Before) != "" {
482 b.WriteString("(\nset -e\n")
483 b.WriteString(job.Before)
484 if !strings.HasSuffix(job.Before, "\n") {
485 b.WriteByte('\n')
486 }
487 b.WriteString(")\nbrc=$?\n")
488 } else {
489 b.WriteString("brc=0\n")
490 }
491 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
492 b.WriteString("echo \"section_start:script:Running script\"\n")
493 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
494 b.WriteString(job.Script)
495 if !strings.HasSuffix(job.Script, "\n") {
496 b.WriteByte('\n')
497 }
498 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
499 b.WriteString("echo \"section_end:script:Running script\"\n")
500 b.WriteString("echo \"section_start:after:After script\"\n")
501 if strings.TrimSpace(job.After) != "" {
502 b.WriteString("(\nset -e\n")
503 b.WriteString(job.After)
504 if !strings.HasSuffix(job.After, "\n") {
505 b.WriteByte('\n')
506 }
507 b.WriteString(")\n")
508 }
509 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
510 return b.String()
511}
512
513// resolveImage kiest het image waarmee de container start.
514// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
515func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
516 image = strings.TrimSpace(image)
517 set, err := images.Effective(r.Presets)
518 if err != nil {
519 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
520 }
521 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
522 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
523 if p, ok := images.Find(set, image); ok {
524 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
525 }
526 return image, dockerImageLines(image, "", nil), missing
527 }
528 p, ok := images.Find(set, image)
529 if !ok {
530 id, digests := r.inspectImage(ctx, image)
531 return image, dockerImageLines(image, id, digests), nil
532 }
533 ref, built, err := p.Ensure(ctx, r.Docker)
534 if err != nil {
535 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
536 }
537 var lines []string
538 if built {
539 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
540 }
541 id, digests := r.inspectImage(ctx, ref)
542 var baseDigests []string
543 if ref != p.From {
544 _, baseDigests = r.inspectImage(ctx, p.From)
545 }
546 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
547 return ref, lines, nil
548}
549
550func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
551 var lines []string
552 if preset != "" {
553 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
554 }
555 if ref == "" {
556 return lines
557 }
558 lines = append(lines, dockerImageLines(ref, id, digests)...)
559 if preset != "" && ref != base {
560 digest := matchingDigest(base, baseDigests)
561 if digest == "" && len(baseDigests) > 0 {
562 digest = strings.TrimSpace(baseDigests[0])
563 }
564 if digest != "" {
565 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
566 } else {
567 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
568 }
569 }
570 return lines
571}
572
573func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
574 if strings.TrimSpace(image) == "" || r.Docker == "" {
575 return "", nil
576 }
577 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
578 if err != nil {
579 return "", nil
580 }
581 return parseImageInspect(out)
582}
583
584func parseImageInspect(out string) (string, []string) {
585 lines := strings.Split(out, "\n")
586 id := ""
587 if len(lines) > 0 {
588 id = strings.TrimSpace(lines[0])
589 }
590 var digests []string
591 for _, line := range lines[1:] {
592 line = strings.TrimSpace(line)
593 if line != "" {
594 digests = append(digests, line)
595 }
596 }
597 return id, digests
598}
599
600func dockerImageLines(image, id string, digests []string) []string {
601 image = strings.TrimSpace(image)
602 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
603 id = strings.TrimSpace(id)
604 digest := matchingDigest(image, digests)
605 switch {
606 case id != "" && digest != "":
607 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
608 case digest != "":
609 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
610 case id != "":
611 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
612 }
613 return lines
614}
615
616func matchingDigest(image string, digests []string) string {
617 name := imageName(image)
618 fallback := ""
619 for _, digest := range digests {
620 digest = strings.TrimSpace(digest)
621 at := strings.LastIndex(digest, "@")
622 if digest == "" || at <= 0 {
623 continue
624 }
625 if fallback == "" {
626 fallback = digest
627 }
628 repo := digest[:at]
629 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
630 return digest
631 }
632 }
633 return fallback
634}
635
636func imageName(image string) string {
637 image = strings.TrimSpace(image)
638 if i := strings.LastIndex(image, "@"); i >= 0 {
639 image = image[:i]
640 }
641 slash := strings.LastIndex(image, "/")
642 name := image
643 if slash >= 0 {
644 name = image[slash+1:]
645 }
646 if i := strings.Index(name, ":"); i >= 0 {
647 name = name[:i]
648 }
649 return name
650}
651
652func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
653 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
654 if err != nil {
655 return err
656 }
657 wanted := map[string]bool{}
658 if job.NeedsSet {
659 for _, n := range job.Needs {
660 wanted[n] = true
661 }
662 } else {
663 stageOrder := map[string]int{}
664 n := 0
665 for _, j := range jobs {
666 if _, ok := stageOrder[j.Stage]; !ok {
667 stageOrder[j.Stage] = n
668 n++
669 }
670 }
671 my := stageOrder[job.Stage]
672 for _, j := range jobs {
673 if stageOrder[j.Stage] < my {
674 wanted[j.Name] = true
675 }
676 }
677 }
678 for _, j := range jobs {
679 if !wanted[j.Name] || j.Status != "success" {
680 continue
681 }
682 for _, rel := range j.ArtifactPaths {
683 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
684 if err != nil {
685 continue
686 }
687 dst := filepath.Join(work, filepath.FromSlash(rel))
688 if err := copyFile(src, dst); err != nil {
689 return err
690 }
691 }
692 }
693 return nil
694}
695
696func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
697 var expires *time.Time
698 if job.ArtifactExpire > 0 {
699 t := time.Now().Add(job.ArtifactExpire)
700 expires = &t
701 }
702 for _, rel := range job.ArtifactPaths {
703 src := filepath.Join(work, filepath.FromSlash(rel))
704 info, err := os.Stat(src)
705 if err != nil || info.IsDir() {
706 return fmt.Errorf("artifact %s ontbreekt", rel)
707 }
708 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
709 if err := copyFile(src, dst); err != nil {
710 return err
711 }
712 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
713 return err
714 }
715 }
716 return nil
717}
718
719func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
720 name, _ := ci.SplitRef(p.Ref)
721 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
722 if err != nil {
723 return nil, nil, err
724 }
725 refProtected := access.BranchProtected(patterns, name)
726 env := map[string]string{}
727 var mask []string
728 if repo.OwnerKind == "group" {
729 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
730 if err != nil {
731 return nil, nil, err
732 }
733 for k, v := range groupSecrets {
734 env[k] = v
735 mask = append(mask, v)
736 }
737 }
738 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
739 if err != nil {
740 return nil, nil, err
741 }
742 for k, v := range repoSecrets {
743 env[k] = v
744 mask = append(mask, v)
745 }
746 for k, v := range job.Variables {
747 env[k] = v
748 }
749 base := strings.TrimRight(r.BaseURL, "/")
750 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
751 var info *ci.MRInfo
752 if p.MRID != 0 {
753 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
754 target := ""
755 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
756 target = strings.TrimSpace(string(out))
757 }
758 info = &ci.MRInfo{
759 ID: m.ID, IID: m.Number, Title: m.Title,
760 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
761 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
762 TargetSHA: target,
763 }
764 }
765 }
766 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
767 pre := ci.Variables(ci.Ref{
768 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
769 DefaultBranch: repo.DefaultBranch, Source: p.Source,
770 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
771 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
772 })
773 for k, v := range pre {
774 env[k] = v
775 }
776 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
777 env["CI_JOB_NAME"] = job.Name
778 env["CI_JOB_STAGE"] = job.Stage
779 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
780 env["CI_JOB_TOKEN"] = token
781 if job.Environment != "" {
782 env["CI_ENVIRONMENT_NAME"] = job.Environment
783 if job.EnvironmentURL != "" {
784 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
785 }
786 }
787 for k, v := range job.Variables {
788 if strings.HasPrefix(k, "CI_NODE_") {
789 env[k] = v
790 }
791 }
792 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
793 // en safe.directory er alsnog bij komt.
794 markWorkspaceSafe(env)
795 mask = append(mask, token)
796 return env, mask, nil
797}
798
799// markWorkspaceSafe zet safe.directory voor elke job.
800// De checkout is van de host-uid; de container-user is dat niet.
801// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
802// * of /workspace zet, blijft zoals hij is.
803func markWorkspaceSafe(env map[string]string) {
804 const key = "safe.directory"
805 n := 0
806 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
807 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
808 n = v
809 }
810 }
811 for i := 0; i < n; i++ {
812 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
813 continue
814 }
815 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
816 case "*", "/workspace":
817 return
818 }
819 }
820 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
821 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
822 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
823}
824
825func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
826 netName := fmt.Sprintf("bluegit-%d", job.ID)
827 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
828 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
829 }
830 var names []string
831 cleanup := func() {
832 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
833 defer cancel()
834 for _, name := range names {
835 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
836 }
837 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
838 }
839 for i, svc := range job.Services {
840 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
841 args := []string{"run", "-d", "--name", cname, "--network", netName}
842 if svc.Alias != "" {
843 args = append(args, "--network-alias", svc.Alias)
844 }
845 cmdEnv := append([]string{}, os.Environ()...)
846 for k, v := range svc.Variables {
847 args = append(args, "-e", k)
848 cmdEnv = append(cmdEnv, k+"="+v)
849 }
850 if len(svc.Entrypoint) > 0 {
851 args = append(args, "--entrypoint", svc.Entrypoint[0])
852 }
853 args = append(args, svc.Name)
854 if len(svc.Entrypoint) > 1 {
855 args = append(args, svc.Entrypoint[1:]...)
856 }
857 args = append(args, svc.Command...)
858 cmd := exec.CommandContext(ctx, r.Docker, args...)
859 cmd.Env = cmdEnv
860 var stderr bytes.Buffer
861 cmd.Stderr = &stderr
862 if err := cmd.Run(); err != nil {
863 cleanup()
864 msg := strings.TrimSpace(stderr.String())
865 if msg == "" {
866 msg = err.Error()
867 }
868 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
869 }
870 names = append(names, cname)
871 }
872 return netName, cleanup, nil
873}
874
875func shouldRetry(job store.Job, kind string) bool {
876 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
877 return false
878 }
879 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
880 return true
881 }
882 if containsStr(job.RetryWhen, kind) {
883 return true
884 }
885 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
886}
887
888func classifyFailure(err error) string {
889 msg := err.Error()
890 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
891 return "runner_system_failure"
892 }
893 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
894 return "job_execution_timeout"
895 }
896 return "script_failure"
897}
898
899func containsStr(list []string, s string) bool {
900 for _, item := range list {
901 if item == s {
902 return true
903 }
904 }
905 return false
906}
907
908func safeSegment(name string) string {
909 var b strings.Builder
910 for _, r := range name {
911 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
912 b.WriteRune(r)
913 } else {
914 b.WriteByte('_')
915 }
916 }
917 if b.Len() == 0 {
918 return "job"
919 }
920 return b.String()
921}
922
923func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
924 if len(job.CachePaths) == 0 {
925 return nil
926 }
927 root := r.cacheDir(repoID, job.CacheKey)
928 for _, rel := range job.CachePaths {
929 src := filepath.Join(root, filepath.FromSlash(rel))
930 if _, err := os.Stat(src); err != nil {
931 continue
932 }
933 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
934 return err
935 }
936 }
937 return nil
938}
939
940func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
941 if len(job.CachePaths) == 0 {
942 return nil
943 }
944 root := r.cacheDir(repoID, job.CacheKey)
945 for _, rel := range job.CachePaths {
946 src := filepath.Join(work, filepath.FromSlash(rel))
947 info, err := os.Stat(src)
948 if err != nil || info.IsDir() {
949 continue
950 }
951 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
952 return err
953 }
954 }
955 return nil
956}
957
958func (r *Runner) cacheDir(repoID int64, key string) string {
959 if key == "" {
960 key = "default"
961 }
962 var b strings.Builder
963 for _, r := range key {
964 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
965 b.WriteRune(r)
966 }
967 }
968 safe := b.String()
969 if safe == "" {
970 safe = "default"
971 }
972 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
973}
974
975func stamp(line string) string {
976 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
977}
978
979type logSink struct {
980 mu sync.Mutex
981 store *store.Store
982 jobID int64
983 secrets []string
984 partial string
985 current string
986 opened time.Time
987}
988
989func (l *logSink) Write(p []byte) (int, error) {
990 l.mu.Lock()
991 defer l.mu.Unlock()
992 l.partial += string(p)
993 for {
994 i := strings.IndexByte(l.partial, '\n')
995 if i < 0 {
996 break
997 }
998 l.emit(strings.TrimRight(l.partial[:i], "\r"))
999 l.partial = l.partial[i+1:]
1000 }
1001 return len(p), nil
1002}
1003
1004func (l *logSink) flush() {
1005 l.mu.Lock()
1006 defer l.mu.Unlock()
1007 if strings.TrimSpace(l.partial) != "" {
1008 l.emit(l.partial)
1009 }
1010 l.partial = ""
1011}
1012
1013func (l *logSink) emit(line string) {
1014 line = ci.Mask(line, l.secrets)
1015 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1016 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1017 dur := int(time.Since(l.opened).Seconds())
1018 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1019 l.current = ""
1020 }
1021 if strings.HasPrefix(line, "section_start:") {
1022 l.current = line
1023 l.opened = time.Now()
1024 }
1025 }
1026 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1027}
1028
1029func (l *logSink) section(name, title string) {
1030 l.emit("section_start:" + name + ":" + title)
1031}
1032
1033func (l *logSink) closeSection() {
1034 if l.current == "" {
1035 return
1036 }
1037 name := "prepare"
1038 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1039 name = parts[0]
1040 }
1041 l.emit("section_end:" + name + ":done")
1042}
1043
1044func copyFile(src, dst string) error {
1045 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1046 return err
1047 }
1048 in, err := os.Open(src)
1049 if err != nil {
1050 return err
1051 }
1052 defer in.Close()
1053 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1054 if err != nil {
1055 return err
1056 }
1057 defer out.Close()
1058 _, err = io.Copy(out, in)
1059 return err
1060}
1061
1062func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1063 if bin == "" {
1064 bin = "git"
1065 }
1066 cmd := exec.CommandContext(ctx, bin, args...)
1067 var stdout, stderr bytes.Buffer
1068 cmd.Stdout = &stdout
1069 cmd.Stderr = &stderr
1070 if err := cmd.Run(); err != nil {
1071 msg := strings.TrimSpace(stderr.String())
1072 if msg == "" {
1073 msg = err.Error()
1074 }
1075 return "", fmt.Errorf("%s", msg)
1076 }
1077 return stdout.String(), nil
1078}