BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

feat: voeg image-presets en een CI-infopagina toe
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
a6677fb6 History
runner.go 27.9 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strings"
15 "sync"
16 "time"
17
18 "gitlab.com/Timfa/bluegit/internal/access"
19 "gitlab.com/Timfa/bluegit/internal/ci"
20 "gitlab.com/Timfa/bluegit/internal/gitrepo"
21 "gitlab.com/Timfa/bluegit/internal/images"
22 "gitlab.com/Timfa/bluegit/internal/store"
23 "gitlab.com/Timfa/bluegit/internal/webhook"
24)
25
26// Runner plant jobs en stuurt het resultaat door naar webhooks.
27type Runner struct {
28 Store *store.Store
29 GitBin string
30 Docker string
31 Storage string
32 BaseURL string
33 Network string
34 Name string
35 Timeout time.Duration
36 Log *slog.Logger
37 OnEvent func(repoID int64, ev webhook.Event)
38 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
39 Presets map[string]images.Override
40 slots chan struct{}
41 mu sync.RWMutex
42 cancelMu sync.Mutex
43 cancel map[int64]context.CancelFunc
44}
45
46// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
47func (r *Runner) Interrupt(ids []int64) {
48 r.cancelMu.Lock()
49 defer r.cancelMu.Unlock()
50 for _, id := range ids {
51 if c := r.cancel[id]; c != nil {
52 c()
53 }
54 }
55}
56
57func (r *Runner) track(id int64, cancel context.CancelFunc) {
58 r.cancelMu.Lock()
59 defer r.cancelMu.Unlock()
60 if r.cancel == nil {
61 r.cancel = map[int64]context.CancelFunc{}
62 }
63 r.cancel[id] = cancel
64}
65
66func (r *Runner) untrack(id int64) {
67 r.cancelMu.Lock()
68 defer r.cancelMu.Unlock()
69 delete(r.cancel, id)
70}
71
72// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
73func (r *Runner) SetStorage(path string) {
74 r.mu.Lock()
75 r.Storage = path
76 r.mu.Unlock()
77}
78
79func (r *Runner) storage() string {
80 r.mu.RLock()
81 defer r.mu.RUnlock()
82 return r.Storage
83}
84
85// Loop blijft jobs oppakken tot ctx afloopt.
86func (r *Runner) Loop(ctx context.Context, concurrency int) {
87 if concurrency < 1 {
88 concurrency = 1
89 }
90 r.slots = make(chan struct{}, concurrency)
91 tick := time.NewTicker(2 * time.Second)
92 defer tick.Stop()
93 for {
94 select {
95 case <-ctx.Done():
96 return
97 case <-tick.C:
98 r.Tick(ctx)
99 }
100 }
101}
102
103// Tick bekijkt alle open pipelines één keer.
104// Zonder opslagpad gebeurt er niets.
105func (r *Runner) Tick(ctx context.Context) {
106 if r.storage() == "" {
107 return
108 }
109 pipes, err := r.Store.ActivePipelines(ctx)
110 if err != nil {
111 r.Log.Error("pipelines", "err", err)
112 return
113 }
114 for _, p := range pipes {
115 r.schedule(ctx, p)
116 }
117}
118
119func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
120 r.settle(ctx, p.ID)
121 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
122 if err != nil {
123 r.Log.Error("jobs", "err", err)
124 return
125 }
126 specs := make([]ci.Job, len(jobs))
127 status := map[string]string{}
128 for i, j := range jobs {
129 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
130 status[j.Name] = j.Status
131 }
132 for _, j := range jobs {
133 if j.Status != "queued" && j.Status != "manual" {
134 continue
135 }
136 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
137 case "skip":
138 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
139 status[j.Name] = "skipped"
140 case "fail":
141 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
142 status[j.Name] = "failed"
143 case "ready":
144 if j.Status != "queued" {
145 continue
146 }
147 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
148 continue
149 }
150 // Meer jobs dan ci.concurrency blijven queued, met een plek
151 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
152 if !r.acquireSlot() {
153 continue
154 }
155 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
156 if err != nil || !ok {
157 r.releaseSlot()
158 continue
159 }
160 status[j.Name] = "running"
161 job := j
162 pipe := p
163 go r.execute(ctx, pipe, job)
164 }
165 }
166}
167
168// settle slaat jobs over die door een harde mislukking niet meer hoeven.
169// Dat gebeurt meteen, ook als de pipelinestatus al op failed was gezet.
170func (r *Runner) settle(ctx context.Context, pipelineID int64) {
171 for {
172 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
173 if err != nil {
174 if r.Log != nil {
175 r.Log.Error("jobs", "err", err)
176 }
177 return
178 }
179 specs := make([]ci.Job, len(jobs))
180 status := map[string]string{}
181 for i, j := range jobs {
182 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
183 status[j.Name] = j.Status
184 }
185 moved := false
186 for _, j := range jobs {
187 if j.Status != "queued" && j.Status != "manual" {
188 continue
189 }
190 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
191 case "skip":
192 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
193 r.Log.Error("skip", "err", err)
194 continue
195 }
196 status[j.Name] = "skipped"
197 moved = true
198 case "fail":
199 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
200 r.Log.Error("fail", "err", err)
201 continue
202 }
203 status[j.Name] = "failed"
204 moved = true
205 }
206 }
207 if !moved {
208 return
209 }
210 }
211}
212
213func indexOf(jobs []store.Job, name string) int {
214 for i, j := range jobs {
215 if j.Name == name {
216 return i
217 }
218 }
219 return 0
220}
221
222func (r *Runner) acquireSlot() bool {
223 if r.slots == nil {
224 return true
225 }
226 select {
227 case r.slots <- struct{}{}:
228 return true
229 default:
230 return false
231 }
232}
233
234func (r *Runner) releaseSlot() {
235 if r.slots == nil {
236 return
237 }
238 select {
239 case <-r.slots:
240 default:
241 }
242}
243
244func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
245 defer r.releaseSlot()
246 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
247 base, stop := context.WithCancel(ctx)
248 r.track(job.ID, stop)
249 defer func() {
250 r.untrack(job.ID)
251 stop()
252 }()
253 runCtx := base
254 limit := r.Timeout
255 if job.Timeout > 0 {
256 limit = job.Timeout
257 }
258 if limit > 0 {
259 var cancel context.CancelFunc
260 runCtx, cancel = context.WithTimeout(base, limit)
261 defer cancel()
262 }
263 mask, err := r.runOne(runCtx, p, job)
264 status := "success"
265 extra := ""
266 kind := ""
267 switch {
268 case err == nil:
269 case ctx.Err() == nil && base.Err() != nil:
270 status = "canceled"
271 extra = stamp("canceled because a newer pipeline started for the same ref")
272 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
273 status = "failed"
274 kind = "job_execution_timeout"
275 extra = stamp(ci.Mask(err.Error(), mask))
276 default:
277 status = "failed"
278 kind = classifyFailure(err)
279 extra = stamp(ci.Mask(err.Error(), mask))
280 }
281 if status == "failed" && shouldRetry(job, kind) {
282 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
283 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
284 r.Log.Error("job opnieuw", "err", ferr)
285 } else if ferr == nil {
286 r.settle(context.Background(), p.ID)
287 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
288 r.Log.Error("pipelinestatus", "err", rerr)
289 }
290 return
291 }
292 }
293 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
294 r.Log.Error("job afronden", "err", ferr)
295 }
296 r.settle(context.Background(), p.ID)
297 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
298 if rerr != nil {
299 r.Log.Error("pipelinestatus", "err", rerr)
300 return
301 }
302 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
303 kind := webhook.EventPipelineSuccess
304 word := "geslaagd"
305 if final == "failed" {
306 kind = webhook.EventPipelineFailure
307 word = "mislukt"
308 }
309 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
310 if err != nil {
311 return
312 }
313 branch, _ := gitrepo.BranchOfRef(p.Ref)
314 if branch == "" {
315 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
316 }
317 r.OnEvent(p.RepoID, webhook.Event{
318 Type: kind,
319 Project: repo.FullPath(),
320 Branch: branch,
321 Commit: p.CommitSHA,
322 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
323 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
324 PipelineID: fmt.Sprint(p.ID),
325 })
326 }
327}
328
329func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
330 repo, err := r.Store.RepoByID(ctx, p.RepoID)
331 if err != nil {
332 return nil, err
333 }
334 sink := &logSink{store: r.Store, jobID: job.ID}
335 sink.section("prepare", "Preparing the docker executor")
336 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
337 _ = os.RemoveAll(work)
338 if err := os.MkdirAll(work, 0o755); err != nil {
339 sink.closeSection()
340 return nil, err
341 }
342 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
343 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
344 sink.closeSection()
345 return nil, fmt.Errorf("clone: %w", err)
346 }
347 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
348 sink.closeSection()
349 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
350 }
351 if err := r.restoreCache(repo.ID, job, work); err != nil {
352 sink.closeSection()
353 return nil, err
354 }
355 if err := r.materialize(ctx, p, job, work); err != nil {
356 sink.closeSection()
357 return nil, err
358 }
359 scriptPath := filepath.Join(work, ".bluegit-job.sh")
360 host, _ := os.Hostname()
361 script := shellScript(job, placeNote{
362 Runner: r.Name, Host: host, WorkDir: "/workspace",
363 PipelineID: p.ID, JobID: job.ID,
364 })
365 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
366 sink.closeSection()
367 return nil, err
368 }
369 token, err := r.Store.CreateJobToken(ctx, job.ID)
370 if err != nil {
371 sink.closeSection()
372 return nil, err
373 }
374 env, mask, err := r.environment(ctx, repo, p, job, token)
375 if err != nil {
376 sink.closeSection()
377 return nil, err
378 }
379 sink.secrets = mask
380 ref, lines, rerr := r.resolveImage(ctx, job.Image)
381 for _, line := range lines {
382 sink.emit(line)
383 }
384 if rerr != nil {
385 sink.closeSection()
386 return mask, rerr
387 }
388 job.Image = ref
389 for i := range job.Services {
390 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
391 for _, line := range slines {
392 sink.emit(line)
393 }
394 if serr != nil {
395 sink.closeSection()
396 return mask, serr
397 }
398 job.Services[i].Name = sref
399 }
400 network := r.Network
401 if len(job.Services) > 0 {
402 net, cleanup, serr := r.startServices(ctx, job)
403 if serr != nil {
404 sink.closeSection()
405 return mask, serr
406 }
407 defer cleanup()
408 network = net
409 }
410 sink.closeSection()
411 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
412 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
413 // staat de waarde niet in de proceslijst.
414 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
415 cmdEnv := append([]string{}, os.Environ()...)
416 for k, v := range env {
417 args = append(args, "-e", k)
418 cmdEnv = append(cmdEnv, k+"="+v)
419 }
420 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
421 cmd := exec.CommandContext(ctx, r.Docker, args...)
422 cmd.Env = cmdEnv
423 cmd.Stdout = sink
424 cmd.Stderr = sink
425 err = cmd.Run()
426 sink.flush()
427 if err != nil {
428 return mask, fmt.Errorf("container ended with an error: %w", err)
429 }
430 if err := r.collect(ctx, job, work); err != nil {
431 return mask, err
432 }
433 if err := r.saveCache(repo.ID, job, work); err != nil {
434 return mask, err
435 }
436 return mask, nil
437}
438
439// placeNote zijn de vaste regels in de sectie Preparing environment.
440type placeNote struct {
441 Runner string
442 Host string
443 WorkDir string
444 PipelineID int64
445 JobID int64
446}
447
448func (n placeNote) lines() []string {
449 runner := strings.TrimSpace(n.Runner)
450 if runner == "" {
451 runner = "Steve"
452 }
453 host := strings.TrimSpace(n.Host)
454 if host == "" {
455 host = "localhost"
456 }
457 dir := strings.TrimSpace(n.WorkDir)
458 if dir == "" {
459 dir = "/workspace"
460 }
461 return []string{
462 fmt.Sprintf("Running on %s via %s...", runner, host),
463 fmt.Sprintf("Working directory: %s", dir),
464 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
465 }
466}
467
468func shSingle(s string) string {
469 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
470}
471
472func shellScript(job store.Job, note placeNote) string {
473 var b strings.Builder
474 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
475 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
476 for _, line := range note.lines() {
477 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
478 }
479 if strings.TrimSpace(job.Before) != "" {
480 b.WriteString("(\nset -e\n")
481 b.WriteString(job.Before)
482 if !strings.HasSuffix(job.Before, "\n") {
483 b.WriteByte('\n')
484 }
485 b.WriteString(")\nbrc=$?\n")
486 } else {
487 b.WriteString("brc=0\n")
488 }
489 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
490 b.WriteString("echo \"section_start:script:Running script\"\n")
491 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
492 b.WriteString(job.Script)
493 if !strings.HasSuffix(job.Script, "\n") {
494 b.WriteByte('\n')
495 }
496 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
497 b.WriteString("echo \"section_end:script:Running script\"\n")
498 b.WriteString("echo \"section_start:after:After script\"\n")
499 if strings.TrimSpace(job.After) != "" {
500 b.WriteString("(\nset -e\n")
501 b.WriteString(job.After)
502 if !strings.HasSuffix(job.After, "\n") {
503 b.WriteByte('\n')
504 }
505 b.WriteString(")\n")
506 }
507 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
508 return b.String()
509}
510
511// resolveImage kiest het image waarmee de container start.
512// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
513func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
514 image = strings.TrimSpace(image)
515 set, err := images.Effective(r.Presets)
516 if err != nil {
517 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
518 }
519 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
520 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
521 if p, ok := images.Find(set, image); ok {
522 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
523 }
524 return image, dockerImageLines(image, "", nil), missing
525 }
526 p, ok := images.Find(set, image)
527 if !ok {
528 id, digests := r.inspectImage(ctx, image)
529 return image, dockerImageLines(image, id, digests), nil
530 }
531 ref, built, err := p.Ensure(ctx, r.Docker)
532 if err != nil {
533 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
534 }
535 var lines []string
536 if built {
537 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
538 }
539 id, digests := r.inspectImage(ctx, ref)
540 var baseDigests []string
541 if ref != p.From {
542 _, baseDigests = r.inspectImage(ctx, p.From)
543 }
544 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
545 return ref, lines, nil
546}
547
548func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
549 var lines []string
550 if preset != "" {
551 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
552 }
553 if ref == "" {
554 return lines
555 }
556 lines = append(lines, dockerImageLines(ref, id, digests)...)
557 if preset != "" && ref != base {
558 digest := matchingDigest(base, baseDigests)
559 if digest == "" && len(baseDigests) > 0 {
560 digest = strings.TrimSpace(baseDigests[0])
561 }
562 if digest != "" {
563 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
564 } else {
565 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
566 }
567 }
568 return lines
569}
570
571func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
572 if strings.TrimSpace(image) == "" || r.Docker == "" {
573 return "", nil
574 }
575 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
576 if err != nil {
577 return "", nil
578 }
579 return parseImageInspect(out)
580}
581
582func parseImageInspect(out string) (string, []string) {
583 lines := strings.Split(out, "\n")
584 id := ""
585 if len(lines) > 0 {
586 id = strings.TrimSpace(lines[0])
587 }
588 var digests []string
589 for _, line := range lines[1:] {
590 line = strings.TrimSpace(line)
591 if line != "" {
592 digests = append(digests, line)
593 }
594 }
595 return id, digests
596}
597
598func dockerImageLines(image, id string, digests []string) []string {
599 image = strings.TrimSpace(image)
600 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
601 id = strings.TrimSpace(id)
602 digest := matchingDigest(image, digests)
603 switch {
604 case id != "" && digest != "":
605 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
606 case digest != "":
607 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
608 case id != "":
609 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
610 }
611 return lines
612}
613
614func matchingDigest(image string, digests []string) string {
615 name := imageName(image)
616 fallback := ""
617 for _, digest := range digests {
618 digest = strings.TrimSpace(digest)
619 at := strings.LastIndex(digest, "@")
620 if digest == "" || at <= 0 {
621 continue
622 }
623 if fallback == "" {
624 fallback = digest
625 }
626 repo := digest[:at]
627 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
628 return digest
629 }
630 }
631 return fallback
632}
633
634func imageName(image string) string {
635 image = strings.TrimSpace(image)
636 if i := strings.LastIndex(image, "@"); i >= 0 {
637 image = image[:i]
638 }
639 slash := strings.LastIndex(image, "/")
640 name := image
641 if slash >= 0 {
642 name = image[slash+1:]
643 }
644 if i := strings.Index(name, ":"); i >= 0 {
645 name = name[:i]
646 }
647 return name
648}
649
650func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
651 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
652 if err != nil {
653 return err
654 }
655 wanted := map[string]bool{}
656 if job.NeedsSet {
657 for _, n := range job.Needs {
658 wanted[n] = true
659 }
660 } else {
661 stageOrder := map[string]int{}
662 n := 0
663 for _, j := range jobs {
664 if _, ok := stageOrder[j.Stage]; !ok {
665 stageOrder[j.Stage] = n
666 n++
667 }
668 }
669 my := stageOrder[job.Stage]
670 for _, j := range jobs {
671 if stageOrder[j.Stage] < my {
672 wanted[j.Name] = true
673 }
674 }
675 }
676 for _, j := range jobs {
677 if !wanted[j.Name] || j.Status != "success" {
678 continue
679 }
680 for _, rel := range j.ArtifactPaths {
681 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
682 if err != nil {
683 continue
684 }
685 dst := filepath.Join(work, filepath.FromSlash(rel))
686 if err := copyFile(src, dst); err != nil {
687 return err
688 }
689 }
690 }
691 return nil
692}
693
694func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
695 var expires *time.Time
696 if job.ArtifactExpire > 0 {
697 t := time.Now().Add(job.ArtifactExpire)
698 expires = &t
699 }
700 for _, rel := range job.ArtifactPaths {
701 src := filepath.Join(work, filepath.FromSlash(rel))
702 info, err := os.Stat(src)
703 if err != nil || info.IsDir() {
704 return fmt.Errorf("artifact %s ontbreekt", rel)
705 }
706 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
707 if err := copyFile(src, dst); err != nil {
708 return err
709 }
710 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
711 return err
712 }
713 }
714 return nil
715}
716
717func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
718 name, _ := ci.SplitRef(p.Ref)
719 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
720 if err != nil {
721 return nil, nil, err
722 }
723 refProtected := access.BranchProtected(patterns, name)
724 env := map[string]string{}
725 var mask []string
726 if repo.OwnerKind == "group" {
727 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
728 if err != nil {
729 return nil, nil, err
730 }
731 for k, v := range groupSecrets {
732 env[k] = v
733 mask = append(mask, v)
734 }
735 }
736 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
737 if err != nil {
738 return nil, nil, err
739 }
740 for k, v := range repoSecrets {
741 env[k] = v
742 mask = append(mask, v)
743 }
744 for k, v := range job.Variables {
745 env[k] = v
746 }
747 base := strings.TrimRight(r.BaseURL, "/")
748 var info *ci.MRInfo
749 if p.MRID != 0 {
750 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
751 target := ""
752 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
753 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
754 target = strings.TrimSpace(string(out))
755 }
756 info = &ci.MRInfo{
757 ID: m.ID, IID: m.Number, Title: m.Title,
758 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
759 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
760 TargetSHA: target,
761 }
762 }
763 }
764 pre := ci.Variables(ci.Ref{
765 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle,
766 DefaultBranch: repo.DefaultBranch, Source: p.Source,
767 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
768 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
769 })
770 for k, v := range pre {
771 env[k] = v
772 }
773 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
774 env["CI_JOB_NAME"] = job.Name
775 env["CI_JOB_STAGE"] = job.Stage
776 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
777 env["CI_JOB_TOKEN"] = token
778 if job.Environment != "" {
779 env["CI_ENVIRONMENT_NAME"] = job.Environment
780 if job.EnvironmentURL != "" {
781 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
782 }
783 }
784 for k, v := range job.Variables {
785 if strings.HasPrefix(k, "CI_NODE_") {
786 env[k] = v
787 }
788 }
789 mask = append(mask, token)
790 return env, mask, nil
791}
792
793func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
794 netName := fmt.Sprintf("bluegit-%d", job.ID)
795 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
796 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
797 }
798 var names []string
799 cleanup := func() {
800 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
801 defer cancel()
802 for _, name := range names {
803 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
804 }
805 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
806 }
807 for i, svc := range job.Services {
808 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
809 args := []string{"run", "-d", "--name", cname, "--network", netName}
810 if svc.Alias != "" {
811 args = append(args, "--network-alias", svc.Alias)
812 }
813 cmdEnv := append([]string{}, os.Environ()...)
814 for k, v := range svc.Variables {
815 args = append(args, "-e", k)
816 cmdEnv = append(cmdEnv, k+"="+v)
817 }
818 if len(svc.Entrypoint) > 0 {
819 args = append(args, "--entrypoint", svc.Entrypoint[0])
820 }
821 args = append(args, svc.Name)
822 if len(svc.Entrypoint) > 1 {
823 args = append(args, svc.Entrypoint[1:]...)
824 }
825 args = append(args, svc.Command...)
826 cmd := exec.CommandContext(ctx, r.Docker, args...)
827 cmd.Env = cmdEnv
828 var stderr bytes.Buffer
829 cmd.Stderr = &stderr
830 if err := cmd.Run(); err != nil {
831 cleanup()
832 msg := strings.TrimSpace(stderr.String())
833 if msg == "" {
834 msg = err.Error()
835 }
836 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
837 }
838 names = append(names, cname)
839 }
840 return netName, cleanup, nil
841}
842
843func shouldRetry(job store.Job, kind string) bool {
844 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
845 return false
846 }
847 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
848 return true
849 }
850 if containsStr(job.RetryWhen, kind) {
851 return true
852 }
853 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
854}
855
856func classifyFailure(err error) string {
857 msg := err.Error()
858 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
859 return "runner_system_failure"
860 }
861 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
862 return "job_execution_timeout"
863 }
864 return "script_failure"
865}
866
867func containsStr(list []string, s string) bool {
868 for _, item := range list {
869 if item == s {
870 return true
871 }
872 }
873 return false
874}
875
876func safeSegment(name string) string {
877 var b strings.Builder
878 for _, r := range name {
879 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
880 b.WriteRune(r)
881 } else {
882 b.WriteByte('_')
883 }
884 }
885 if b.Len() == 0 {
886 return "job"
887 }
888 return b.String()
889}
890
891func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
892 if len(job.CachePaths) == 0 {
893 return nil
894 }
895 root := r.cacheDir(repoID, job.CacheKey)
896 for _, rel := range job.CachePaths {
897 src := filepath.Join(root, filepath.FromSlash(rel))
898 if _, err := os.Stat(src); err != nil {
899 continue
900 }
901 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
902 return err
903 }
904 }
905 return nil
906}
907
908func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
909 if len(job.CachePaths) == 0 {
910 return nil
911 }
912 root := r.cacheDir(repoID, job.CacheKey)
913 for _, rel := range job.CachePaths {
914 src := filepath.Join(work, filepath.FromSlash(rel))
915 info, err := os.Stat(src)
916 if err != nil || info.IsDir() {
917 continue
918 }
919 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
920 return err
921 }
922 }
923 return nil
924}
925
926func (r *Runner) cacheDir(repoID int64, key string) string {
927 if key == "" {
928 key = "default"
929 }
930 var b strings.Builder
931 for _, r := range key {
932 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
933 b.WriteRune(r)
934 }
935 }
936 safe := b.String()
937 if safe == "" {
938 safe = "default"
939 }
940 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
941}
942
943func stamp(line string) string {
944 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
945}
946
947type logSink struct {
948 mu sync.Mutex
949 store *store.Store
950 jobID int64
951 secrets []string
952 partial string
953 current string
954 opened time.Time
955}
956
957func (l *logSink) Write(p []byte) (int, error) {
958 l.mu.Lock()
959 defer l.mu.Unlock()
960 l.partial += string(p)
961 for {
962 i := strings.IndexByte(l.partial, '\n')
963 if i < 0 {
964 break
965 }
966 l.emit(strings.TrimRight(l.partial[:i], "\r"))
967 l.partial = l.partial[i+1:]
968 }
969 return len(p), nil
970}
971
972func (l *logSink) flush() {
973 l.mu.Lock()
974 defer l.mu.Unlock()
975 if strings.TrimSpace(l.partial) != "" {
976 l.emit(l.partial)
977 }
978 l.partial = ""
979}
980
981func (l *logSink) emit(line string) {
982 line = ci.Mask(line, l.secrets)
983 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
984 if l.current != "" && strings.HasPrefix(line, "section_end:") {
985 dur := int(time.Since(l.opened).Seconds())
986 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
987 l.current = ""
988 }
989 if strings.HasPrefix(line, "section_start:") {
990 l.current = line
991 l.opened = time.Now()
992 }
993 }
994 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
995}
996
997func (l *logSink) section(name, title string) {
998 l.emit("section_start:" + name + ":" + title)
999}
1000
1001func (l *logSink) closeSection() {
1002 if l.current == "" {
1003 return
1004 }
1005 name := "prepare"
1006 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1007 name = parts[0]
1008 }
1009 l.emit("section_end:" + name + ":done")
1010}
1011
1012func copyFile(src, dst string) error {
1013 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1014 return err
1015 }
1016 in, err := os.Open(src)
1017 if err != nil {
1018 return err
1019 }
1020 defer in.Close()
1021 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1022 if err != nil {
1023 return err
1024 }
1025 defer out.Close()
1026 _, err = io.Copy(out, in)
1027 return err
1028}
1029
1030func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1031 if bin == "" {
1032 bin = "git"
1033 }
1034 cmd := exec.CommandContext(ctx, bin, args...)
1035 var stdout, stderr bytes.Buffer
1036 cmd.Stdout = &stdout
1037 cmd.Stderr = &stderr
1038 if err := cmd.Run(); err != nil {
1039 msg := strings.TrimSpace(stderr.String())
1040 if msg == "" {
1041 msg = err.Error()
1042 }
1043 return "", fmt.Errorf("%s", msg)
1044 }
1045 return stdout.String(), nil
1046}