BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

fix: sla een job over als een need faalt of is overgeslagen
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
5d24c9de History
runner.go 27.9 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strings"
15 "sync"
16 "time"
17
18 "gitlab.com/Timfa/bluegit/internal/access"
19 "gitlab.com/Timfa/bluegit/internal/ci"
20 "gitlab.com/Timfa/bluegit/internal/gitrepo"
21 "gitlab.com/Timfa/bluegit/internal/images"
22 "gitlab.com/Timfa/bluegit/internal/store"
23 "gitlab.com/Timfa/bluegit/internal/webhook"
24)
25
26// Runner plant jobs en stuurt het resultaat door naar webhooks.
27type Runner struct {
28 Store *store.Store
29 GitBin string
30 Docker string
31 Storage string
32 BaseURL string
33 Network string
34 Name string
35 Timeout time.Duration
36 Log *slog.Logger
37 OnEvent func(repoID int64, ev webhook.Event)
38 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
39 Presets map[string]images.Override
40 slots chan struct{}
41 mu sync.RWMutex
42 cancelMu sync.Mutex
43 cancel map[int64]context.CancelFunc
44}
45
46// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
47func (r *Runner) Interrupt(ids []int64) {
48 r.cancelMu.Lock()
49 defer r.cancelMu.Unlock()
50 for _, id := range ids {
51 if c := r.cancel[id]; c != nil {
52 c()
53 }
54 }
55}
56
57func (r *Runner) track(id int64, cancel context.CancelFunc) {
58 r.cancelMu.Lock()
59 defer r.cancelMu.Unlock()
60 if r.cancel == nil {
61 r.cancel = map[int64]context.CancelFunc{}
62 }
63 r.cancel[id] = cancel
64}
65
66func (r *Runner) untrack(id int64) {
67 r.cancelMu.Lock()
68 defer r.cancelMu.Unlock()
69 delete(r.cancel, id)
70}
71
72// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
73func (r *Runner) SetStorage(path string) {
74 r.mu.Lock()
75 r.Storage = path
76 r.mu.Unlock()
77}
78
79func (r *Runner) storage() string {
80 r.mu.RLock()
81 defer r.mu.RUnlock()
82 return r.Storage
83}
84
85// Loop blijft jobs oppakken tot ctx afloopt.
86func (r *Runner) Loop(ctx context.Context, concurrency int) {
87 if concurrency < 1 {
88 concurrency = 1
89 }
90 r.slots = make(chan struct{}, concurrency)
91 tick := time.NewTicker(2 * time.Second)
92 defer tick.Stop()
93 for {
94 select {
95 case <-ctx.Done():
96 return
97 case <-tick.C:
98 r.Tick(ctx)
99 }
100 }
101}
102
103// Tick bekijkt alle open pipelines één keer.
104// Zonder opslagpad gebeurt er niets.
105func (r *Runner) Tick(ctx context.Context) {
106 if r.storage() == "" {
107 return
108 }
109 pipes, err := r.Store.ActivePipelines(ctx)
110 if err != nil {
111 r.Log.Error("pipelines", "err", err)
112 return
113 }
114 for _, p := range pipes {
115 r.schedule(ctx, p)
116 }
117}
118
119func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
120 r.settle(ctx, p.ID)
121 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
122 if err != nil {
123 r.Log.Error("jobs", "err", err)
124 return
125 }
126 specs := make([]ci.Job, len(jobs))
127 status := map[string]string{}
128 for i, j := range jobs {
129 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
130 status[j.Name] = j.Status
131 }
132 for _, j := range jobs {
133 if j.Status != "queued" && j.Status != "manual" {
134 continue
135 }
136 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
137 case "skip":
138 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
139 status[j.Name] = "skipped"
140 case "fail":
141 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
142 status[j.Name] = "failed"
143 case "ready":
144 if j.Status != "queued" {
145 continue
146 }
147 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
148 continue
149 }
150 // Meer jobs dan ci.concurrency blijven queued, met een plek
151 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
152 if !r.acquireSlot() {
153 continue
154 }
155 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
156 if err != nil || !ok {
157 r.releaseSlot()
158 continue
159 }
160 status[j.Name] = "running"
161 job := j
162 pipe := p
163 go r.execute(ctx, pipe, job)
164 }
165 }
166}
167
168// settle slaat jobs over die door een harde mislukking of een
169// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
170// pipelinestatus al op failed was gezet.
171func (r *Runner) settle(ctx context.Context, pipelineID int64) {
172 for {
173 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
174 if err != nil {
175 if r.Log != nil {
176 r.Log.Error("jobs", "err", err)
177 }
178 return
179 }
180 specs := make([]ci.Job, len(jobs))
181 status := map[string]string{}
182 for i, j := range jobs {
183 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
184 status[j.Name] = j.Status
185 }
186 moved := false
187 for _, j := range jobs {
188 if j.Status != "queued" && j.Status != "manual" {
189 continue
190 }
191 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
192 case "skip":
193 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
194 r.Log.Error("skip", "err", err)
195 continue
196 }
197 status[j.Name] = "skipped"
198 moved = true
199 case "fail":
200 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
201 r.Log.Error("fail", "err", err)
202 continue
203 }
204 status[j.Name] = "failed"
205 moved = true
206 }
207 }
208 if !moved {
209 return
210 }
211 }
212}
213
214func indexOf(jobs []store.Job, name string) int {
215 for i, j := range jobs {
216 if j.Name == name {
217 return i
218 }
219 }
220 return 0
221}
222
223func (r *Runner) acquireSlot() bool {
224 if r.slots == nil {
225 return true
226 }
227 select {
228 case r.slots <- struct{}{}:
229 return true
230 default:
231 return false
232 }
233}
234
235func (r *Runner) releaseSlot() {
236 if r.slots == nil {
237 return
238 }
239 select {
240 case <-r.slots:
241 default:
242 }
243}
244
245func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
246 defer r.releaseSlot()
247 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
248 base, stop := context.WithCancel(ctx)
249 r.track(job.ID, stop)
250 defer func() {
251 r.untrack(job.ID)
252 stop()
253 }()
254 runCtx := base
255 limit := r.Timeout
256 if job.Timeout > 0 {
257 limit = job.Timeout
258 }
259 if limit > 0 {
260 var cancel context.CancelFunc
261 runCtx, cancel = context.WithTimeout(base, limit)
262 defer cancel()
263 }
264 mask, err := r.runOne(runCtx, p, job)
265 status := "success"
266 extra := ""
267 kind := ""
268 switch {
269 case err == nil:
270 case ctx.Err() == nil && base.Err() != nil:
271 status = "canceled"
272 extra = stamp("canceled because a newer pipeline started for the same ref")
273 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
274 status = "failed"
275 kind = "job_execution_timeout"
276 extra = stamp(ci.Mask(err.Error(), mask))
277 default:
278 status = "failed"
279 kind = classifyFailure(err)
280 extra = stamp(ci.Mask(err.Error(), mask))
281 }
282 if status == "failed" && shouldRetry(job, kind) {
283 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
284 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
285 r.Log.Error("job opnieuw", "err", ferr)
286 } else if ferr == nil {
287 r.settle(context.Background(), p.ID)
288 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
289 r.Log.Error("pipelinestatus", "err", rerr)
290 }
291 return
292 }
293 }
294 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
295 r.Log.Error("job afronden", "err", ferr)
296 }
297 r.settle(context.Background(), p.ID)
298 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
299 if rerr != nil {
300 r.Log.Error("pipelinestatus", "err", rerr)
301 return
302 }
303 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
304 kind := webhook.EventPipelineSuccess
305 word := "geslaagd"
306 if final == "failed" {
307 kind = webhook.EventPipelineFailure
308 word = "mislukt"
309 }
310 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
311 if err != nil {
312 return
313 }
314 branch, _ := gitrepo.BranchOfRef(p.Ref)
315 if branch == "" {
316 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
317 }
318 r.OnEvent(p.RepoID, webhook.Event{
319 Type: kind,
320 Project: repo.FullPath(),
321 Branch: branch,
322 Commit: p.CommitSHA,
323 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
324 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
325 PipelineID: fmt.Sprint(p.ID),
326 })
327 }
328}
329
330func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
331 repo, err := r.Store.RepoByID(ctx, p.RepoID)
332 if err != nil {
333 return nil, err
334 }
335 sink := &logSink{store: r.Store, jobID: job.ID}
336 sink.section("prepare", "Preparing the docker executor")
337 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
338 _ = os.RemoveAll(work)
339 if err := os.MkdirAll(work, 0o755); err != nil {
340 sink.closeSection()
341 return nil, err
342 }
343 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
344 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
345 sink.closeSection()
346 return nil, fmt.Errorf("clone: %w", err)
347 }
348 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
349 sink.closeSection()
350 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
351 }
352 if err := r.restoreCache(repo.ID, job, work); err != nil {
353 sink.closeSection()
354 return nil, err
355 }
356 if err := r.materialize(ctx, p, job, work); err != nil {
357 sink.closeSection()
358 return nil, err
359 }
360 scriptPath := filepath.Join(work, ".bluegit-job.sh")
361 host, _ := os.Hostname()
362 script := shellScript(job, placeNote{
363 Runner: r.Name, Host: host, WorkDir: "/workspace",
364 PipelineID: p.ID, JobID: job.ID,
365 })
366 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
367 sink.closeSection()
368 return nil, err
369 }
370 token, err := r.Store.CreateJobToken(ctx, job.ID)
371 if err != nil {
372 sink.closeSection()
373 return nil, err
374 }
375 env, mask, err := r.environment(ctx, repo, p, job, token)
376 if err != nil {
377 sink.closeSection()
378 return nil, err
379 }
380 sink.secrets = mask
381 ref, lines, rerr := r.resolveImage(ctx, job.Image)
382 for _, line := range lines {
383 sink.emit(line)
384 }
385 if rerr != nil {
386 sink.closeSection()
387 return mask, rerr
388 }
389 job.Image = ref
390 for i := range job.Services {
391 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
392 for _, line := range slines {
393 sink.emit(line)
394 }
395 if serr != nil {
396 sink.closeSection()
397 return mask, serr
398 }
399 job.Services[i].Name = sref
400 }
401 network := r.Network
402 if len(job.Services) > 0 {
403 net, cleanup, serr := r.startServices(ctx, job)
404 if serr != nil {
405 sink.closeSection()
406 return mask, serr
407 }
408 defer cleanup()
409 network = net
410 }
411 sink.closeSection()
412 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
413 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
414 // staat de waarde niet in de proceslijst.
415 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
416 cmdEnv := append([]string{}, os.Environ()...)
417 for k, v := range env {
418 args = append(args, "-e", k)
419 cmdEnv = append(cmdEnv, k+"="+v)
420 }
421 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
422 cmd := exec.CommandContext(ctx, r.Docker, args...)
423 cmd.Env = cmdEnv
424 cmd.Stdout = sink
425 cmd.Stderr = sink
426 err = cmd.Run()
427 sink.flush()
428 if err != nil {
429 return mask, fmt.Errorf("container ended with an error: %w", err)
430 }
431 if err := r.collect(ctx, job, work); err != nil {
432 return mask, err
433 }
434 if err := r.saveCache(repo.ID, job, work); err != nil {
435 return mask, err
436 }
437 return mask, nil
438}
439
440// placeNote zijn de vaste regels in de sectie Preparing environment.
441type placeNote struct {
442 Runner string
443 Host string
444 WorkDir string
445 PipelineID int64
446 JobID int64
447}
448
449func (n placeNote) lines() []string {
450 runner := strings.TrimSpace(n.Runner)
451 if runner == "" {
452 runner = "Steve"
453 }
454 host := strings.TrimSpace(n.Host)
455 if host == "" {
456 host = "localhost"
457 }
458 dir := strings.TrimSpace(n.WorkDir)
459 if dir == "" {
460 dir = "/workspace"
461 }
462 return []string{
463 fmt.Sprintf("Running on %s via %s...", runner, host),
464 fmt.Sprintf("Working directory: %s", dir),
465 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
466 }
467}
468
469func shSingle(s string) string {
470 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
471}
472
473func shellScript(job store.Job, note placeNote) string {
474 var b strings.Builder
475 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
476 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
477 for _, line := range note.lines() {
478 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
479 }
480 if strings.TrimSpace(job.Before) != "" {
481 b.WriteString("(\nset -e\n")
482 b.WriteString(job.Before)
483 if !strings.HasSuffix(job.Before, "\n") {
484 b.WriteByte('\n')
485 }
486 b.WriteString(")\nbrc=$?\n")
487 } else {
488 b.WriteString("brc=0\n")
489 }
490 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
491 b.WriteString("echo \"section_start:script:Running script\"\n")
492 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
493 b.WriteString(job.Script)
494 if !strings.HasSuffix(job.Script, "\n") {
495 b.WriteByte('\n')
496 }
497 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
498 b.WriteString("echo \"section_end:script:Running script\"\n")
499 b.WriteString("echo \"section_start:after:After script\"\n")
500 if strings.TrimSpace(job.After) != "" {
501 b.WriteString("(\nset -e\n")
502 b.WriteString(job.After)
503 if !strings.HasSuffix(job.After, "\n") {
504 b.WriteByte('\n')
505 }
506 b.WriteString(")\n")
507 }
508 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
509 return b.String()
510}
511
512// resolveImage kiest het image waarmee de container start.
513// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
514func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
515 image = strings.TrimSpace(image)
516 set, err := images.Effective(r.Presets)
517 if err != nil {
518 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
519 }
520 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
521 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
522 if p, ok := images.Find(set, image); ok {
523 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
524 }
525 return image, dockerImageLines(image, "", nil), missing
526 }
527 p, ok := images.Find(set, image)
528 if !ok {
529 id, digests := r.inspectImage(ctx, image)
530 return image, dockerImageLines(image, id, digests), nil
531 }
532 ref, built, err := p.Ensure(ctx, r.Docker)
533 if err != nil {
534 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
535 }
536 var lines []string
537 if built {
538 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
539 }
540 id, digests := r.inspectImage(ctx, ref)
541 var baseDigests []string
542 if ref != p.From {
543 _, baseDigests = r.inspectImage(ctx, p.From)
544 }
545 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
546 return ref, lines, nil
547}
548
549func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
550 var lines []string
551 if preset != "" {
552 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
553 }
554 if ref == "" {
555 return lines
556 }
557 lines = append(lines, dockerImageLines(ref, id, digests)...)
558 if preset != "" && ref != base {
559 digest := matchingDigest(base, baseDigests)
560 if digest == "" && len(baseDigests) > 0 {
561 digest = strings.TrimSpace(baseDigests[0])
562 }
563 if digest != "" {
564 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
565 } else {
566 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
567 }
568 }
569 return lines
570}
571
572func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
573 if strings.TrimSpace(image) == "" || r.Docker == "" {
574 return "", nil
575 }
576 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
577 if err != nil {
578 return "", nil
579 }
580 return parseImageInspect(out)
581}
582
583func parseImageInspect(out string) (string, []string) {
584 lines := strings.Split(out, "\n")
585 id := ""
586 if len(lines) > 0 {
587 id = strings.TrimSpace(lines[0])
588 }
589 var digests []string
590 for _, line := range lines[1:] {
591 line = strings.TrimSpace(line)
592 if line != "" {
593 digests = append(digests, line)
594 }
595 }
596 return id, digests
597}
598
599func dockerImageLines(image, id string, digests []string) []string {
600 image = strings.TrimSpace(image)
601 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
602 id = strings.TrimSpace(id)
603 digest := matchingDigest(image, digests)
604 switch {
605 case id != "" && digest != "":
606 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
607 case digest != "":
608 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
609 case id != "":
610 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
611 }
612 return lines
613}
614
615func matchingDigest(image string, digests []string) string {
616 name := imageName(image)
617 fallback := ""
618 for _, digest := range digests {
619 digest = strings.TrimSpace(digest)
620 at := strings.LastIndex(digest, "@")
621 if digest == "" || at <= 0 {
622 continue
623 }
624 if fallback == "" {
625 fallback = digest
626 }
627 repo := digest[:at]
628 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
629 return digest
630 }
631 }
632 return fallback
633}
634
635func imageName(image string) string {
636 image = strings.TrimSpace(image)
637 if i := strings.LastIndex(image, "@"); i >= 0 {
638 image = image[:i]
639 }
640 slash := strings.LastIndex(image, "/")
641 name := image
642 if slash >= 0 {
643 name = image[slash+1:]
644 }
645 if i := strings.Index(name, ":"); i >= 0 {
646 name = name[:i]
647 }
648 return name
649}
650
651func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
652 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
653 if err != nil {
654 return err
655 }
656 wanted := map[string]bool{}
657 if job.NeedsSet {
658 for _, n := range job.Needs {
659 wanted[n] = true
660 }
661 } else {
662 stageOrder := map[string]int{}
663 n := 0
664 for _, j := range jobs {
665 if _, ok := stageOrder[j.Stage]; !ok {
666 stageOrder[j.Stage] = n
667 n++
668 }
669 }
670 my := stageOrder[job.Stage]
671 for _, j := range jobs {
672 if stageOrder[j.Stage] < my {
673 wanted[j.Name] = true
674 }
675 }
676 }
677 for _, j := range jobs {
678 if !wanted[j.Name] || j.Status != "success" {
679 continue
680 }
681 for _, rel := range j.ArtifactPaths {
682 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
683 if err != nil {
684 continue
685 }
686 dst := filepath.Join(work, filepath.FromSlash(rel))
687 if err := copyFile(src, dst); err != nil {
688 return err
689 }
690 }
691 }
692 return nil
693}
694
695func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
696 var expires *time.Time
697 if job.ArtifactExpire > 0 {
698 t := time.Now().Add(job.ArtifactExpire)
699 expires = &t
700 }
701 for _, rel := range job.ArtifactPaths {
702 src := filepath.Join(work, filepath.FromSlash(rel))
703 info, err := os.Stat(src)
704 if err != nil || info.IsDir() {
705 return fmt.Errorf("artifact %s ontbreekt", rel)
706 }
707 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
708 if err := copyFile(src, dst); err != nil {
709 return err
710 }
711 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
712 return err
713 }
714 }
715 return nil
716}
717
718func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
719 name, _ := ci.SplitRef(p.Ref)
720 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
721 if err != nil {
722 return nil, nil, err
723 }
724 refProtected := access.BranchProtected(patterns, name)
725 env := map[string]string{}
726 var mask []string
727 if repo.OwnerKind == "group" {
728 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
729 if err != nil {
730 return nil, nil, err
731 }
732 for k, v := range groupSecrets {
733 env[k] = v
734 mask = append(mask, v)
735 }
736 }
737 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
738 if err != nil {
739 return nil, nil, err
740 }
741 for k, v := range repoSecrets {
742 env[k] = v
743 mask = append(mask, v)
744 }
745 for k, v := range job.Variables {
746 env[k] = v
747 }
748 base := strings.TrimRight(r.BaseURL, "/")
749 var info *ci.MRInfo
750 if p.MRID != 0 {
751 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
752 target := ""
753 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
754 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
755 target = strings.TrimSpace(string(out))
756 }
757 info = &ci.MRInfo{
758 ID: m.ID, IID: m.Number, Title: m.Title,
759 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
760 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
761 TargetSHA: target,
762 }
763 }
764 }
765 pre := ci.Variables(ci.Ref{
766 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle,
767 DefaultBranch: repo.DefaultBranch, Source: p.Source,
768 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
769 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
770 })
771 for k, v := range pre {
772 env[k] = v
773 }
774 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
775 env["CI_JOB_NAME"] = job.Name
776 env["CI_JOB_STAGE"] = job.Stage
777 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
778 env["CI_JOB_TOKEN"] = token
779 if job.Environment != "" {
780 env["CI_ENVIRONMENT_NAME"] = job.Environment
781 if job.EnvironmentURL != "" {
782 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
783 }
784 }
785 for k, v := range job.Variables {
786 if strings.HasPrefix(k, "CI_NODE_") {
787 env[k] = v
788 }
789 }
790 mask = append(mask, token)
791 return env, mask, nil
792}
793
794func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
795 netName := fmt.Sprintf("bluegit-%d", job.ID)
796 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
797 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
798 }
799 var names []string
800 cleanup := func() {
801 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
802 defer cancel()
803 for _, name := range names {
804 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
805 }
806 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
807 }
808 for i, svc := range job.Services {
809 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
810 args := []string{"run", "-d", "--name", cname, "--network", netName}
811 if svc.Alias != "" {
812 args = append(args, "--network-alias", svc.Alias)
813 }
814 cmdEnv := append([]string{}, os.Environ()...)
815 for k, v := range svc.Variables {
816 args = append(args, "-e", k)
817 cmdEnv = append(cmdEnv, k+"="+v)
818 }
819 if len(svc.Entrypoint) > 0 {
820 args = append(args, "--entrypoint", svc.Entrypoint[0])
821 }
822 args = append(args, svc.Name)
823 if len(svc.Entrypoint) > 1 {
824 args = append(args, svc.Entrypoint[1:]...)
825 }
826 args = append(args, svc.Command...)
827 cmd := exec.CommandContext(ctx, r.Docker, args...)
828 cmd.Env = cmdEnv
829 var stderr bytes.Buffer
830 cmd.Stderr = &stderr
831 if err := cmd.Run(); err != nil {
832 cleanup()
833 msg := strings.TrimSpace(stderr.String())
834 if msg == "" {
835 msg = err.Error()
836 }
837 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
838 }
839 names = append(names, cname)
840 }
841 return netName, cleanup, nil
842}
843
844func shouldRetry(job store.Job, kind string) bool {
845 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
846 return false
847 }
848 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
849 return true
850 }
851 if containsStr(job.RetryWhen, kind) {
852 return true
853 }
854 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
855}
856
857func classifyFailure(err error) string {
858 msg := err.Error()
859 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
860 return "runner_system_failure"
861 }
862 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
863 return "job_execution_timeout"
864 }
865 return "script_failure"
866}
867
868func containsStr(list []string, s string) bool {
869 for _, item := range list {
870 if item == s {
871 return true
872 }
873 }
874 return false
875}
876
877func safeSegment(name string) string {
878 var b strings.Builder
879 for _, r := range name {
880 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
881 b.WriteRune(r)
882 } else {
883 b.WriteByte('_')
884 }
885 }
886 if b.Len() == 0 {
887 return "job"
888 }
889 return b.String()
890}
891
892func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
893 if len(job.CachePaths) == 0 {
894 return nil
895 }
896 root := r.cacheDir(repoID, job.CacheKey)
897 for _, rel := range job.CachePaths {
898 src := filepath.Join(root, filepath.FromSlash(rel))
899 if _, err := os.Stat(src); err != nil {
900 continue
901 }
902 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
903 return err
904 }
905 }
906 return nil
907}
908
909func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
910 if len(job.CachePaths) == 0 {
911 return nil
912 }
913 root := r.cacheDir(repoID, job.CacheKey)
914 for _, rel := range job.CachePaths {
915 src := filepath.Join(work, filepath.FromSlash(rel))
916 info, err := os.Stat(src)
917 if err != nil || info.IsDir() {
918 continue
919 }
920 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
921 return err
922 }
923 }
924 return nil
925}
926
927func (r *Runner) cacheDir(repoID int64, key string) string {
928 if key == "" {
929 key = "default"
930 }
931 var b strings.Builder
932 for _, r := range key {
933 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
934 b.WriteRune(r)
935 }
936 }
937 safe := b.String()
938 if safe == "" {
939 safe = "default"
940 }
941 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
942}
943
944func stamp(line string) string {
945 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
946}
947
948type logSink struct {
949 mu sync.Mutex
950 store *store.Store
951 jobID int64
952 secrets []string
953 partial string
954 current string
955 opened time.Time
956}
957
958func (l *logSink) Write(p []byte) (int, error) {
959 l.mu.Lock()
960 defer l.mu.Unlock()
961 l.partial += string(p)
962 for {
963 i := strings.IndexByte(l.partial, '\n')
964 if i < 0 {
965 break
966 }
967 l.emit(strings.TrimRight(l.partial[:i], "\r"))
968 l.partial = l.partial[i+1:]
969 }
970 return len(p), nil
971}
972
973func (l *logSink) flush() {
974 l.mu.Lock()
975 defer l.mu.Unlock()
976 if strings.TrimSpace(l.partial) != "" {
977 l.emit(l.partial)
978 }
979 l.partial = ""
980}
981
982func (l *logSink) emit(line string) {
983 line = ci.Mask(line, l.secrets)
984 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
985 if l.current != "" && strings.HasPrefix(line, "section_end:") {
986 dur := int(time.Since(l.opened).Seconds())
987 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
988 l.current = ""
989 }
990 if strings.HasPrefix(line, "section_start:") {
991 l.current = line
992 l.opened = time.Now()
993 }
994 }
995 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
996}
997
998func (l *logSink) section(name, title string) {
999 l.emit("section_start:" + name + ":" + title)
1000}
1001
1002func (l *logSink) closeSection() {
1003 if l.current == "" {
1004 return
1005 }
1006 name := "prepare"
1007 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1008 name = parts[0]
1009 }
1010 l.emit("section_end:" + name + ":done")
1011}
1012
1013func copyFile(src, dst string) error {
1014 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1015 return err
1016 }
1017 in, err := os.Open(src)
1018 if err != nil {
1019 return err
1020 }
1021 defer in.Close()
1022 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1023 if err != nil {
1024 return err
1025 }
1026 defer out.Close()
1027 _, err = io.Copy(out, in)
1028 return err
1029}
1030
1031func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1032 if bin == "" {
1033 bin = "git"
1034 }
1035 cmd := exec.CommandContext(ctx, bin, args...)
1036 var stdout, stderr bytes.Buffer
1037 cmd.Stdout = &stdout
1038 cmd.Stderr = &stderr
1039 if err := cmd.Run(); err != nil {
1040 msg := strings.TrimSpace(stderr.String())
1041 if msg == "" {
1042 msg = err.Error()
1043 }
1044 return "", fmt.Errorf("%s", msg)
1045 }
1046 return stdout.String(), nil
1047}