BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

fix: ververs actieve pipelines en toon de wachtrijplek
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
1cf0f0da History
runner.go 25.7 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strings"
15 "sync"
16 "time"
17
18 "gitlab.com/Timfa/bluegit/internal/access"
19 "gitlab.com/Timfa/bluegit/internal/ci"
20 "gitlab.com/Timfa/bluegit/internal/gitrepo"
21 "gitlab.com/Timfa/bluegit/internal/store"
22 "gitlab.com/Timfa/bluegit/internal/webhook"
23)
24
25// Runner plant jobs en stuurt het resultaat door naar webhooks.
26type Runner struct {
27 Store *store.Store
28 GitBin string
29 Docker string
30 Storage string
31 BaseURL string
32 Network string
33 Name string
34 Timeout time.Duration
35 Log *slog.Logger
36 OnEvent func(repoID int64, ev webhook.Event)
37 slots chan struct{}
38 mu sync.RWMutex
39 cancelMu sync.Mutex
40 cancel map[int64]context.CancelFunc
41}
42
43// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
44func (r *Runner) Interrupt(ids []int64) {
45 r.cancelMu.Lock()
46 defer r.cancelMu.Unlock()
47 for _, id := range ids {
48 if c := r.cancel[id]; c != nil {
49 c()
50 }
51 }
52}
53
54func (r *Runner) track(id int64, cancel context.CancelFunc) {
55 r.cancelMu.Lock()
56 defer r.cancelMu.Unlock()
57 if r.cancel == nil {
58 r.cancel = map[int64]context.CancelFunc{}
59 }
60 r.cancel[id] = cancel
61}
62
63func (r *Runner) untrack(id int64) {
64 r.cancelMu.Lock()
65 defer r.cancelMu.Unlock()
66 delete(r.cancel, id)
67}
68
69// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
70func (r *Runner) SetStorage(path string) {
71 r.mu.Lock()
72 r.Storage = path
73 r.mu.Unlock()
74}
75
76func (r *Runner) storage() string {
77 r.mu.RLock()
78 defer r.mu.RUnlock()
79 return r.Storage
80}
81
82// Loop blijft jobs oppakken tot ctx afloopt.
83func (r *Runner) Loop(ctx context.Context, concurrency int) {
84 if concurrency < 1 {
85 concurrency = 1
86 }
87 r.slots = make(chan struct{}, concurrency)
88 tick := time.NewTicker(2 * time.Second)
89 defer tick.Stop()
90 for {
91 select {
92 case <-ctx.Done():
93 return
94 case <-tick.C:
95 r.Tick(ctx)
96 }
97 }
98}
99
100// Tick bekijkt alle open pipelines één keer.
101// Zonder opslagpad gebeurt er niets.
102func (r *Runner) Tick(ctx context.Context) {
103 if r.storage() == "" {
104 return
105 }
106 pipes, err := r.Store.ActivePipelines(ctx)
107 if err != nil {
108 r.Log.Error("pipelines", "err", err)
109 return
110 }
111 for _, p := range pipes {
112 r.schedule(ctx, p)
113 }
114}
115
116func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
117 r.settle(ctx, p.ID)
118 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
119 if err != nil {
120 r.Log.Error("jobs", "err", err)
121 return
122 }
123 specs := make([]ci.Job, len(jobs))
124 status := map[string]string{}
125 for i, j := range jobs {
126 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
127 status[j.Name] = j.Status
128 }
129 for _, j := range jobs {
130 if j.Status != "queued" && j.Status != "manual" {
131 continue
132 }
133 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
134 case "skip":
135 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
136 status[j.Name] = "skipped"
137 case "fail":
138 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
139 status[j.Name] = "failed"
140 case "ready":
141 if j.Status != "queued" {
142 continue
143 }
144 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
145 continue
146 }
147 // Meer jobs dan ci.concurrency blijven queued, met een plek
148 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
149 if !r.acquireSlot() {
150 continue
151 }
152 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
153 if err != nil || !ok {
154 r.releaseSlot()
155 continue
156 }
157 status[j.Name] = "running"
158 job := j
159 pipe := p
160 go r.execute(ctx, pipe, job)
161 }
162 }
163}
164
165// settle slaat jobs over die door een harde mislukking niet meer hoeven.
166// Dat gebeurt meteen, ook als de pipelinestatus al op failed was gezet.
167func (r *Runner) settle(ctx context.Context, pipelineID int64) {
168 for {
169 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
170 if err != nil {
171 if r.Log != nil {
172 r.Log.Error("jobs", "err", err)
173 }
174 return
175 }
176 specs := make([]ci.Job, len(jobs))
177 status := map[string]string{}
178 for i, j := range jobs {
179 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
180 status[j.Name] = j.Status
181 }
182 moved := false
183 for _, j := range jobs {
184 if j.Status != "queued" && j.Status != "manual" {
185 continue
186 }
187 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
188 case "skip":
189 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
190 r.Log.Error("skip", "err", err)
191 continue
192 }
193 status[j.Name] = "skipped"
194 moved = true
195 case "fail":
196 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
197 r.Log.Error("fail", "err", err)
198 continue
199 }
200 status[j.Name] = "failed"
201 moved = true
202 }
203 }
204 if !moved {
205 return
206 }
207 }
208}
209
210func indexOf(jobs []store.Job, name string) int {
211 for i, j := range jobs {
212 if j.Name == name {
213 return i
214 }
215 }
216 return 0
217}
218
219func (r *Runner) acquireSlot() bool {
220 if r.slots == nil {
221 return true
222 }
223 select {
224 case r.slots <- struct{}{}:
225 return true
226 default:
227 return false
228 }
229}
230
231func (r *Runner) releaseSlot() {
232 if r.slots == nil {
233 return
234 }
235 select {
236 case <-r.slots:
237 default:
238 }
239}
240
241func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
242 defer r.releaseSlot()
243 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
244 base, stop := context.WithCancel(ctx)
245 r.track(job.ID, stop)
246 defer func() {
247 r.untrack(job.ID)
248 stop()
249 }()
250 runCtx := base
251 limit := r.Timeout
252 if job.Timeout > 0 {
253 limit = job.Timeout
254 }
255 if limit > 0 {
256 var cancel context.CancelFunc
257 runCtx, cancel = context.WithTimeout(base, limit)
258 defer cancel()
259 }
260 mask, err := r.runOne(runCtx, p, job)
261 status := "success"
262 extra := ""
263 kind := ""
264 switch {
265 case err == nil:
266 case ctx.Err() == nil && base.Err() != nil:
267 status = "canceled"
268 extra = stamp("canceled because a newer pipeline started for the same ref")
269 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
270 status = "failed"
271 kind = "job_execution_timeout"
272 extra = stamp(ci.Mask(err.Error(), mask))
273 default:
274 status = "failed"
275 kind = classifyFailure(err)
276 extra = stamp(ci.Mask(err.Error(), mask))
277 }
278 if status == "failed" && shouldRetry(job, kind) {
279 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
280 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
281 r.Log.Error("job opnieuw", "err", ferr)
282 } else if ferr == nil {
283 r.settle(context.Background(), p.ID)
284 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
285 r.Log.Error("pipelinestatus", "err", rerr)
286 }
287 return
288 }
289 }
290 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
291 r.Log.Error("job afronden", "err", ferr)
292 }
293 r.settle(context.Background(), p.ID)
294 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
295 if rerr != nil {
296 r.Log.Error("pipelinestatus", "err", rerr)
297 return
298 }
299 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
300 kind := webhook.EventPipelineSuccess
301 word := "geslaagd"
302 if final == "failed" {
303 kind = webhook.EventPipelineFailure
304 word = "mislukt"
305 }
306 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
307 if err != nil {
308 return
309 }
310 branch, _ := gitrepo.BranchOfRef(p.Ref)
311 if branch == "" {
312 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
313 }
314 r.OnEvent(p.RepoID, webhook.Event{
315 Type: kind,
316 Project: repo.FullPath(),
317 Branch: branch,
318 Commit: p.CommitSHA,
319 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
320 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
321 PipelineID: fmt.Sprint(p.ID),
322 })
323 }
324}
325
326func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
327 repo, err := r.Store.RepoByID(ctx, p.RepoID)
328 if err != nil {
329 return nil, err
330 }
331 sink := &logSink{store: r.Store, jobID: job.ID}
332 sink.section("prepare", "Preparing the docker executor")
333 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
334 _ = os.RemoveAll(work)
335 if err := os.MkdirAll(work, 0o755); err != nil {
336 sink.closeSection()
337 return nil, err
338 }
339 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
340 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
341 sink.closeSection()
342 return nil, fmt.Errorf("clone: %w", err)
343 }
344 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
345 sink.closeSection()
346 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
347 }
348 if err := r.restoreCache(repo.ID, job, work); err != nil {
349 sink.closeSection()
350 return nil, err
351 }
352 if err := r.materialize(ctx, p, job, work); err != nil {
353 sink.closeSection()
354 return nil, err
355 }
356 scriptPath := filepath.Join(work, ".bluegit-job.sh")
357 host, _ := os.Hostname()
358 script := shellScript(job, placeNote{
359 Runner: r.Name, Host: host, WorkDir: "/workspace",
360 PipelineID: p.ID, JobID: job.ID,
361 })
362 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
363 sink.closeSection()
364 return nil, err
365 }
366 token, err := r.Store.CreateJobToken(ctx, job.ID)
367 if err != nil {
368 sink.closeSection()
369 return nil, err
370 }
371 env, mask, err := r.environment(ctx, repo, p, job, token)
372 if err != nil {
373 sink.closeSection()
374 return nil, err
375 }
376 sink.secrets = mask
377 if _, err := exec.LookPath(r.Docker); err != nil {
378 for _, line := range dockerImageLines(job.Image, "", nil) {
379 sink.emit(line)
380 }
381 sink.closeSection()
382 return mask, fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
383 }
384 r.logImage(ctx, sink, job.Image)
385 network := r.Network
386 if len(job.Services) > 0 {
387 net, cleanup, serr := r.startServices(ctx, job)
388 if serr != nil {
389 sink.closeSection()
390 return mask, serr
391 }
392 defer cleanup()
393 network = net
394 }
395 sink.closeSection()
396 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
397 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
398 // staat de waarde niet in de proceslijst.
399 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
400 cmdEnv := append([]string{}, os.Environ()...)
401 for k, v := range env {
402 args = append(args, "-e", k)
403 cmdEnv = append(cmdEnv, k+"="+v)
404 }
405 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
406 cmd := exec.CommandContext(ctx, r.Docker, args...)
407 cmd.Env = cmdEnv
408 cmd.Stdout = sink
409 cmd.Stderr = sink
410 err = cmd.Run()
411 sink.flush()
412 if err != nil {
413 return mask, fmt.Errorf("container ended with an error: %w", err)
414 }
415 if err := r.collect(ctx, job, work); err != nil {
416 return mask, err
417 }
418 if err := r.saveCache(repo.ID, job, work); err != nil {
419 return mask, err
420 }
421 return mask, nil
422}
423
424// placeNote zijn de vaste regels in de sectie Preparing environment.
425type placeNote struct {
426 Runner string
427 Host string
428 WorkDir string
429 PipelineID int64
430 JobID int64
431}
432
433func (n placeNote) lines() []string {
434 runner := strings.TrimSpace(n.Runner)
435 if runner == "" {
436 runner = "Steve"
437 }
438 host := strings.TrimSpace(n.Host)
439 if host == "" {
440 host = "localhost"
441 }
442 dir := strings.TrimSpace(n.WorkDir)
443 if dir == "" {
444 dir = "/workspace"
445 }
446 return []string{
447 fmt.Sprintf("Running on %s via %s...", runner, host),
448 fmt.Sprintf("Working directory: %s", dir),
449 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
450 }
451}
452
453func shSingle(s string) string {
454 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
455}
456
457func shellScript(job store.Job, note placeNote) string {
458 var b strings.Builder
459 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
460 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
461 for _, line := range note.lines() {
462 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
463 }
464 if strings.TrimSpace(job.Before) != "" {
465 b.WriteString("(\nset -e\n")
466 b.WriteString(job.Before)
467 if !strings.HasSuffix(job.Before, "\n") {
468 b.WriteByte('\n')
469 }
470 b.WriteString(")\nbrc=$?\n")
471 } else {
472 b.WriteString("brc=0\n")
473 }
474 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
475 b.WriteString("echo \"section_start:script:Running script\"\n")
476 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
477 b.WriteString(job.Script)
478 if !strings.HasSuffix(job.Script, "\n") {
479 b.WriteByte('\n')
480 }
481 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
482 b.WriteString("echo \"section_end:script:Running script\"\n")
483 b.WriteString("echo \"section_start:after:After script\"\n")
484 if strings.TrimSpace(job.After) != "" {
485 b.WriteString("(\nset -e\n")
486 b.WriteString(job.After)
487 if !strings.HasSuffix(job.After, "\n") {
488 b.WriteByte('\n')
489 }
490 b.WriteString(")\n")
491 }
492 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
493 return b.String()
494}
495
496func (r *Runner) logImage(ctx context.Context, sink *logSink, image string) {
497 id, digests := r.inspectImage(ctx, image)
498 for _, line := range dockerImageLines(image, id, digests) {
499 sink.emit(line)
500 }
501}
502
503func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
504 if strings.TrimSpace(image) == "" || r.Docker == "" {
505 return "", nil
506 }
507 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
508 if err != nil {
509 return "", nil
510 }
511 return parseImageInspect(out)
512}
513
514func parseImageInspect(out string) (string, []string) {
515 lines := strings.Split(out, "\n")
516 id := ""
517 if len(lines) > 0 {
518 id = strings.TrimSpace(lines[0])
519 }
520 var digests []string
521 for _, line := range lines[1:] {
522 line = strings.TrimSpace(line)
523 if line != "" {
524 digests = append(digests, line)
525 }
526 }
527 return id, digests
528}
529
530func dockerImageLines(image, id string, digests []string) []string {
531 image = strings.TrimSpace(image)
532 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
533 id = strings.TrimSpace(id)
534 digest := matchingDigest(image, digests)
535 switch {
536 case id != "" && digest != "":
537 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
538 case digest != "":
539 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
540 case id != "":
541 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
542 }
543 return lines
544}
545
546func matchingDigest(image string, digests []string) string {
547 name := imageName(image)
548 fallback := ""
549 for _, digest := range digests {
550 digest = strings.TrimSpace(digest)
551 at := strings.LastIndex(digest, "@")
552 if digest == "" || at <= 0 {
553 continue
554 }
555 if fallback == "" {
556 fallback = digest
557 }
558 repo := digest[:at]
559 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
560 return digest
561 }
562 }
563 return fallback
564}
565
566func imageName(image string) string {
567 image = strings.TrimSpace(image)
568 if i := strings.LastIndex(image, "@"); i >= 0 {
569 image = image[:i]
570 }
571 slash := strings.LastIndex(image, "/")
572 name := image
573 if slash >= 0 {
574 name = image[slash+1:]
575 }
576 if i := strings.Index(name, ":"); i >= 0 {
577 name = name[:i]
578 }
579 return name
580}
581
582func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
583 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
584 if err != nil {
585 return err
586 }
587 wanted := map[string]bool{}
588 if job.NeedsSet {
589 for _, n := range job.Needs {
590 wanted[n] = true
591 }
592 } else {
593 stageOrder := map[string]int{}
594 n := 0
595 for _, j := range jobs {
596 if _, ok := stageOrder[j.Stage]; !ok {
597 stageOrder[j.Stage] = n
598 n++
599 }
600 }
601 my := stageOrder[job.Stage]
602 for _, j := range jobs {
603 if stageOrder[j.Stage] < my {
604 wanted[j.Name] = true
605 }
606 }
607 }
608 for _, j := range jobs {
609 if !wanted[j.Name] || j.Status != "success" {
610 continue
611 }
612 for _, rel := range j.ArtifactPaths {
613 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
614 if err != nil {
615 continue
616 }
617 dst := filepath.Join(work, filepath.FromSlash(rel))
618 if err := copyFile(src, dst); err != nil {
619 return err
620 }
621 }
622 }
623 return nil
624}
625
626func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
627 var expires *time.Time
628 if job.ArtifactExpire > 0 {
629 t := time.Now().Add(job.ArtifactExpire)
630 expires = &t
631 }
632 for _, rel := range job.ArtifactPaths {
633 src := filepath.Join(work, filepath.FromSlash(rel))
634 info, err := os.Stat(src)
635 if err != nil || info.IsDir() {
636 return fmt.Errorf("artifact %s ontbreekt", rel)
637 }
638 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
639 if err := copyFile(src, dst); err != nil {
640 return err
641 }
642 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
643 return err
644 }
645 }
646 return nil
647}
648
649func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
650 name, _ := ci.SplitRef(p.Ref)
651 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
652 if err != nil {
653 return nil, nil, err
654 }
655 refProtected := access.BranchProtected(patterns, name)
656 env := map[string]string{}
657 var mask []string
658 if repo.OwnerKind == "group" {
659 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
660 if err != nil {
661 return nil, nil, err
662 }
663 for k, v := range groupSecrets {
664 env[k] = v
665 mask = append(mask, v)
666 }
667 }
668 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
669 if err != nil {
670 return nil, nil, err
671 }
672 for k, v := range repoSecrets {
673 env[k] = v
674 mask = append(mask, v)
675 }
676 for k, v := range job.Variables {
677 env[k] = v
678 }
679 base := strings.TrimRight(r.BaseURL, "/")
680 var info *ci.MRInfo
681 if p.MRID != 0 {
682 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
683 target := ""
684 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
685 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
686 target = strings.TrimSpace(string(out))
687 }
688 info = &ci.MRInfo{
689 ID: m.ID, IID: m.Number, Title: m.Title,
690 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
691 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
692 TargetSHA: target,
693 }
694 }
695 }
696 pre := ci.Variables(ci.Ref{
697 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle,
698 DefaultBranch: repo.DefaultBranch, Source: p.Source,
699 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
700 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
701 })
702 for k, v := range pre {
703 env[k] = v
704 }
705 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
706 env["CI_JOB_NAME"] = job.Name
707 env["CI_JOB_STAGE"] = job.Stage
708 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
709 env["CI_JOB_TOKEN"] = token
710 if job.Environment != "" {
711 env["CI_ENVIRONMENT_NAME"] = job.Environment
712 if job.EnvironmentURL != "" {
713 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
714 }
715 }
716 for k, v := range job.Variables {
717 if strings.HasPrefix(k, "CI_NODE_") {
718 env[k] = v
719 }
720 }
721 mask = append(mask, token)
722 return env, mask, nil
723}
724
725func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
726 netName := fmt.Sprintf("bluegit-%d", job.ID)
727 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
728 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
729 }
730 var names []string
731 cleanup := func() {
732 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
733 defer cancel()
734 for _, name := range names {
735 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
736 }
737 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
738 }
739 for i, svc := range job.Services {
740 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
741 args := []string{"run", "-d", "--name", cname, "--network", netName}
742 if svc.Alias != "" {
743 args = append(args, "--network-alias", svc.Alias)
744 }
745 cmdEnv := append([]string{}, os.Environ()...)
746 for k, v := range svc.Variables {
747 args = append(args, "-e", k)
748 cmdEnv = append(cmdEnv, k+"="+v)
749 }
750 if len(svc.Entrypoint) > 0 {
751 args = append(args, "--entrypoint", svc.Entrypoint[0])
752 }
753 args = append(args, svc.Name)
754 if len(svc.Entrypoint) > 1 {
755 args = append(args, svc.Entrypoint[1:]...)
756 }
757 args = append(args, svc.Command...)
758 cmd := exec.CommandContext(ctx, r.Docker, args...)
759 cmd.Env = cmdEnv
760 var stderr bytes.Buffer
761 cmd.Stderr = &stderr
762 if err := cmd.Run(); err != nil {
763 cleanup()
764 msg := strings.TrimSpace(stderr.String())
765 if msg == "" {
766 msg = err.Error()
767 }
768 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
769 }
770 names = append(names, cname)
771 }
772 return netName, cleanup, nil
773}
774
775func shouldRetry(job store.Job, kind string) bool {
776 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
777 return false
778 }
779 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
780 return true
781 }
782 if containsStr(job.RetryWhen, kind) {
783 return true
784 }
785 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
786}
787
788func classifyFailure(err error) string {
789 msg := err.Error()
790 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
791 return "runner_system_failure"
792 }
793 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
794 return "job_execution_timeout"
795 }
796 return "script_failure"
797}
798
799func containsStr(list []string, s string) bool {
800 for _, item := range list {
801 if item == s {
802 return true
803 }
804 }
805 return false
806}
807
808func safeSegment(name string) string {
809 var b strings.Builder
810 for _, r := range name {
811 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
812 b.WriteRune(r)
813 } else {
814 b.WriteByte('_')
815 }
816 }
817 if b.Len() == 0 {
818 return "job"
819 }
820 return b.String()
821}
822
823func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
824 if len(job.CachePaths) == 0 {
825 return nil
826 }
827 root := r.cacheDir(repoID, job.CacheKey)
828 for _, rel := range job.CachePaths {
829 src := filepath.Join(root, filepath.FromSlash(rel))
830 if _, err := os.Stat(src); err != nil {
831 continue
832 }
833 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
834 return err
835 }
836 }
837 return nil
838}
839
840func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
841 if len(job.CachePaths) == 0 {
842 return nil
843 }
844 root := r.cacheDir(repoID, job.CacheKey)
845 for _, rel := range job.CachePaths {
846 src := filepath.Join(work, filepath.FromSlash(rel))
847 info, err := os.Stat(src)
848 if err != nil || info.IsDir() {
849 continue
850 }
851 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
852 return err
853 }
854 }
855 return nil
856}
857
858func (r *Runner) cacheDir(repoID int64, key string) string {
859 if key == "" {
860 key = "default"
861 }
862 var b strings.Builder
863 for _, r := range key {
864 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
865 b.WriteRune(r)
866 }
867 }
868 safe := b.String()
869 if safe == "" {
870 safe = "default"
871 }
872 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
873}
874
875func stamp(line string) string {
876 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
877}
878
879type logSink struct {
880 mu sync.Mutex
881 store *store.Store
882 jobID int64
883 secrets []string
884 partial string
885 current string
886 opened time.Time
887}
888
889func (l *logSink) Write(p []byte) (int, error) {
890 l.mu.Lock()
891 defer l.mu.Unlock()
892 l.partial += string(p)
893 for {
894 i := strings.IndexByte(l.partial, '\n')
895 if i < 0 {
896 break
897 }
898 l.emit(strings.TrimRight(l.partial[:i], "\r"))
899 l.partial = l.partial[i+1:]
900 }
901 return len(p), nil
902}
903
904func (l *logSink) flush() {
905 l.mu.Lock()
906 defer l.mu.Unlock()
907 if strings.TrimSpace(l.partial) != "" {
908 l.emit(l.partial)
909 }
910 l.partial = ""
911}
912
913func (l *logSink) emit(line string) {
914 line = ci.Mask(line, l.secrets)
915 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
916 if l.current != "" && strings.HasPrefix(line, "section_end:") {
917 dur := int(time.Since(l.opened).Seconds())
918 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
919 l.current = ""
920 }
921 if strings.HasPrefix(line, "section_start:") {
922 l.current = line
923 l.opened = time.Now()
924 }
925 }
926 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
927}
928
929func (l *logSink) section(name, title string) {
930 l.emit("section_start:" + name + ":" + title)
931}
932
933func (l *logSink) closeSection() {
934 if l.current == "" {
935 return
936 }
937 name := "prepare"
938 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
939 name = parts[0]
940 }
941 l.emit("section_end:" + name + ":done")
942}
943
944func copyFile(src, dst string) error {
945 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
946 return err
947 }
948 in, err := os.Open(src)
949 if err != nil {
950 return err
951 }
952 defer in.Close()
953 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
954 if err != nil {
955 return err
956 }
957 defer out.Close()
958 _, err = io.Copy(out, in)
959 return err
960}
961
962func execOut(ctx context.Context, bin string, args ...string) (string, error) {
963 if bin == "" {
964 bin = "git"
965 }
966 cmd := exec.CommandContext(ctx, bin, args...)
967 var stdout, stderr bytes.Buffer
968 cmd.Stdout = &stdout
969 cmd.Stderr = &stderr
970 if err := cmd.Run(); err != nil {
971 msg := strings.TrimSpace(stderr.String())
972 if msg == "" {
973 msg = err.Error()
974 }
975 return "", fmt.Errorf("%s", msg)
976 }
977 return stdout.String(), nil
978}