BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

fix: laat popovers zweven en breek joblogs standaard af
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
f8889dcf History
runner.go 25.3 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strings"
15 "sync"
16 "time"
17
18 "gitlab.com/Timfa/bluegit/internal/access"
19 "gitlab.com/Timfa/bluegit/internal/ci"
20 "gitlab.com/Timfa/bluegit/internal/gitrepo"
21 "gitlab.com/Timfa/bluegit/internal/store"
22 "gitlab.com/Timfa/bluegit/internal/webhook"
23)
24
25// Runner plant jobs en stuurt het resultaat door naar webhooks.
26type Runner struct {
27 Store *store.Store
28 GitBin string
29 Docker string
30 Storage string
31 BaseURL string
32 Network string
33 Name string
34 Timeout time.Duration
35 Log *slog.Logger
36 OnEvent func(repoID int64, ev webhook.Event)
37 slots chan struct{}
38 mu sync.RWMutex
39 cancelMu sync.Mutex
40 cancel map[int64]context.CancelFunc
41}
42
43// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
44func (r *Runner) Interrupt(ids []int64) {
45 r.cancelMu.Lock()
46 defer r.cancelMu.Unlock()
47 for _, id := range ids {
48 if c := r.cancel[id]; c != nil {
49 c()
50 }
51 }
52}
53
54func (r *Runner) track(id int64, cancel context.CancelFunc) {
55 r.cancelMu.Lock()
56 defer r.cancelMu.Unlock()
57 if r.cancel == nil {
58 r.cancel = map[int64]context.CancelFunc{}
59 }
60 r.cancel[id] = cancel
61}
62
63func (r *Runner) untrack(id int64) {
64 r.cancelMu.Lock()
65 defer r.cancelMu.Unlock()
66 delete(r.cancel, id)
67}
68
69// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
70func (r *Runner) SetStorage(path string) {
71 r.mu.Lock()
72 r.Storage = path
73 r.mu.Unlock()
74}
75
76func (r *Runner) storage() string {
77 r.mu.RLock()
78 defer r.mu.RUnlock()
79 return r.Storage
80}
81
82// Loop blijft jobs oppakken tot ctx afloopt.
83func (r *Runner) Loop(ctx context.Context, concurrency int) {
84 if concurrency < 1 {
85 concurrency = 1
86 }
87 r.slots = make(chan struct{}, concurrency)
88 tick := time.NewTicker(2 * time.Second)
89 defer tick.Stop()
90 for {
91 select {
92 case <-ctx.Done():
93 return
94 case <-tick.C:
95 r.Tick(ctx)
96 }
97 }
98}
99
100// Tick bekijkt alle open pipelines één keer.
101// Zonder opslagpad gebeurt er niets.
102func (r *Runner) Tick(ctx context.Context) {
103 if r.storage() == "" {
104 return
105 }
106 pipes, err := r.Store.ActivePipelines(ctx)
107 if err != nil {
108 r.Log.Error("pipelines", "err", err)
109 return
110 }
111 for _, p := range pipes {
112 r.schedule(ctx, p)
113 }
114}
115
116func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
117 r.settle(ctx, p.ID)
118 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
119 if err != nil {
120 r.Log.Error("jobs", "err", err)
121 return
122 }
123 specs := make([]ci.Job, len(jobs))
124 status := map[string]string{}
125 for i, j := range jobs {
126 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
127 status[j.Name] = j.Status
128 }
129 for _, j := range jobs {
130 if j.Status != "queued" && j.Status != "manual" {
131 continue
132 }
133 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
134 case "skip":
135 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
136 status[j.Name] = "skipped"
137 case "fail":
138 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
139 status[j.Name] = "failed"
140 case "ready":
141 if j.Status != "queued" {
142 continue
143 }
144 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
145 continue
146 }
147 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
148 if err != nil || !ok {
149 continue
150 }
151 status[j.Name] = "running"
152 job := j
153 pipe := p
154 go r.execute(ctx, pipe, job)
155 }
156 }
157}
158
159// settle slaat jobs over die door een harde mislukking niet meer hoeven.
160// Dat gebeurt meteen, ook als de pipelinestatus al op failed was gezet.
161func (r *Runner) settle(ctx context.Context, pipelineID int64) {
162 for {
163 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
164 if err != nil {
165 if r.Log != nil {
166 r.Log.Error("jobs", "err", err)
167 }
168 return
169 }
170 specs := make([]ci.Job, len(jobs))
171 status := map[string]string{}
172 for i, j := range jobs {
173 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
174 status[j.Name] = j.Status
175 }
176 moved := false
177 for _, j := range jobs {
178 if j.Status != "queued" && j.Status != "manual" {
179 continue
180 }
181 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
182 case "skip":
183 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
184 r.Log.Error("skip", "err", err)
185 continue
186 }
187 status[j.Name] = "skipped"
188 moved = true
189 case "fail":
190 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
191 r.Log.Error("fail", "err", err)
192 continue
193 }
194 status[j.Name] = "failed"
195 moved = true
196 }
197 }
198 if !moved {
199 return
200 }
201 }
202}
203
204func indexOf(jobs []store.Job, name string) int {
205 for i, j := range jobs {
206 if j.Name == name {
207 return i
208 }
209 }
210 return 0
211}
212
213func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
214 if r.slots != nil {
215 r.slots <- struct{}{}
216 defer func() { <-r.slots }()
217 }
218 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
219 base, stop := context.WithCancel(ctx)
220 r.track(job.ID, stop)
221 defer func() {
222 r.untrack(job.ID)
223 stop()
224 }()
225 runCtx := base
226 limit := r.Timeout
227 if job.Timeout > 0 {
228 limit = job.Timeout
229 }
230 if limit > 0 {
231 var cancel context.CancelFunc
232 runCtx, cancel = context.WithTimeout(base, limit)
233 defer cancel()
234 }
235 mask, err := r.runOne(runCtx, p, job)
236 status := "success"
237 extra := ""
238 kind := ""
239 switch {
240 case err == nil:
241 case ctx.Err() == nil && base.Err() != nil:
242 status = "canceled"
243 extra = stamp("canceled because a newer pipeline started for the same ref")
244 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
245 status = "failed"
246 kind = "job_execution_timeout"
247 extra = stamp(ci.Mask(err.Error(), mask))
248 default:
249 status = "failed"
250 kind = classifyFailure(err)
251 extra = stamp(ci.Mask(err.Error(), mask))
252 }
253 if status == "failed" && shouldRetry(job, kind) {
254 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
255 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
256 r.Log.Error("job opnieuw", "err", ferr)
257 } else if ferr == nil {
258 r.settle(context.Background(), p.ID)
259 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
260 r.Log.Error("pipelinestatus", "err", rerr)
261 }
262 return
263 }
264 }
265 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
266 r.Log.Error("job afronden", "err", ferr)
267 }
268 r.settle(context.Background(), p.ID)
269 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
270 if rerr != nil {
271 r.Log.Error("pipelinestatus", "err", rerr)
272 return
273 }
274 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
275 kind := webhook.EventPipelineSuccess
276 word := "geslaagd"
277 if final == "failed" {
278 kind = webhook.EventPipelineFailure
279 word = "mislukt"
280 }
281 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
282 if err != nil {
283 return
284 }
285 branch, _ := gitrepo.BranchOfRef(p.Ref)
286 if branch == "" {
287 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
288 }
289 r.OnEvent(p.RepoID, webhook.Event{
290 Type: kind,
291 Project: repo.FullPath(),
292 Branch: branch,
293 Commit: p.CommitSHA,
294 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
295 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
296 PipelineID: fmt.Sprint(p.ID),
297 })
298 }
299}
300
301func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
302 repo, err := r.Store.RepoByID(ctx, p.RepoID)
303 if err != nil {
304 return nil, err
305 }
306 sink := &logSink{store: r.Store, jobID: job.ID}
307 sink.section("prepare", "Preparing the docker executor")
308 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
309 _ = os.RemoveAll(work)
310 if err := os.MkdirAll(work, 0o755); err != nil {
311 sink.closeSection()
312 return nil, err
313 }
314 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
315 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
316 sink.closeSection()
317 return nil, fmt.Errorf("clone: %w", err)
318 }
319 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
320 sink.closeSection()
321 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
322 }
323 if err := r.restoreCache(repo.ID, job, work); err != nil {
324 sink.closeSection()
325 return nil, err
326 }
327 if err := r.materialize(ctx, p, job, work); err != nil {
328 sink.closeSection()
329 return nil, err
330 }
331 scriptPath := filepath.Join(work, ".bluegit-job.sh")
332 host, _ := os.Hostname()
333 script := shellScript(job, placeNote{
334 Runner: r.Name, Host: host, WorkDir: "/workspace",
335 PipelineID: p.ID, JobID: job.ID,
336 })
337 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
338 sink.closeSection()
339 return nil, err
340 }
341 token, err := r.Store.CreateJobToken(ctx, job.ID)
342 if err != nil {
343 sink.closeSection()
344 return nil, err
345 }
346 env, mask, err := r.environment(ctx, repo, p, job, token)
347 if err != nil {
348 sink.closeSection()
349 return nil, err
350 }
351 sink.secrets = mask
352 if _, err := exec.LookPath(r.Docker); err != nil {
353 for _, line := range dockerImageLines(job.Image, "", nil) {
354 sink.emit(line)
355 }
356 sink.closeSection()
357 return mask, fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
358 }
359 r.logImage(ctx, sink, job.Image)
360 network := r.Network
361 if len(job.Services) > 0 {
362 net, cleanup, serr := r.startServices(ctx, job)
363 if serr != nil {
364 sink.closeSection()
365 return mask, serr
366 }
367 defer cleanup()
368 network = net
369 }
370 sink.closeSection()
371 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
372 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
373 // staat de waarde niet in de proceslijst.
374 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
375 cmdEnv := append([]string{}, os.Environ()...)
376 for k, v := range env {
377 args = append(args, "-e", k)
378 cmdEnv = append(cmdEnv, k+"="+v)
379 }
380 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
381 cmd := exec.CommandContext(ctx, r.Docker, args...)
382 cmd.Env = cmdEnv
383 cmd.Stdout = sink
384 cmd.Stderr = sink
385 err = cmd.Run()
386 sink.flush()
387 if err != nil {
388 return mask, fmt.Errorf("container ended with an error: %w", err)
389 }
390 if err := r.collect(ctx, job, work); err != nil {
391 return mask, err
392 }
393 if err := r.saveCache(repo.ID, job, work); err != nil {
394 return mask, err
395 }
396 return mask, nil
397}
398
399// placeNote zijn de vaste regels in de sectie Preparing environment.
400type placeNote struct {
401 Runner string
402 Host string
403 WorkDir string
404 PipelineID int64
405 JobID int64
406}
407
408func (n placeNote) lines() []string {
409 runner := strings.TrimSpace(n.Runner)
410 if runner == "" {
411 runner = "Steve"
412 }
413 host := strings.TrimSpace(n.Host)
414 if host == "" {
415 host = "localhost"
416 }
417 dir := strings.TrimSpace(n.WorkDir)
418 if dir == "" {
419 dir = "/workspace"
420 }
421 return []string{
422 fmt.Sprintf("Running on %s via %s...", runner, host),
423 fmt.Sprintf("Working directory: %s", dir),
424 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
425 }
426}
427
428func shSingle(s string) string {
429 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
430}
431
432func shellScript(job store.Job, note placeNote) string {
433 var b strings.Builder
434 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
435 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
436 for _, line := range note.lines() {
437 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
438 }
439 if strings.TrimSpace(job.Before) != "" {
440 b.WriteString("(\nset -e\n")
441 b.WriteString(job.Before)
442 if !strings.HasSuffix(job.Before, "\n") {
443 b.WriteByte('\n')
444 }
445 b.WriteString(")\nbrc=$?\n")
446 } else {
447 b.WriteString("brc=0\n")
448 }
449 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
450 b.WriteString("echo \"section_start:script:Running script\"\n")
451 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
452 b.WriteString(job.Script)
453 if !strings.HasSuffix(job.Script, "\n") {
454 b.WriteByte('\n')
455 }
456 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
457 b.WriteString("echo \"section_end:script:Running script\"\n")
458 b.WriteString("echo \"section_start:after:After script\"\n")
459 if strings.TrimSpace(job.After) != "" {
460 b.WriteString("(\nset -e\n")
461 b.WriteString(job.After)
462 if !strings.HasSuffix(job.After, "\n") {
463 b.WriteByte('\n')
464 }
465 b.WriteString(")\n")
466 }
467 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
468 return b.String()
469}
470
471func (r *Runner) logImage(ctx context.Context, sink *logSink, image string) {
472 id, digests := r.inspectImage(ctx, image)
473 for _, line := range dockerImageLines(image, id, digests) {
474 sink.emit(line)
475 }
476}
477
478func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
479 if strings.TrimSpace(image) == "" || r.Docker == "" {
480 return "", nil
481 }
482 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
483 if err != nil {
484 return "", nil
485 }
486 return parseImageInspect(out)
487}
488
489func parseImageInspect(out string) (string, []string) {
490 lines := strings.Split(out, "\n")
491 id := ""
492 if len(lines) > 0 {
493 id = strings.TrimSpace(lines[0])
494 }
495 var digests []string
496 for _, line := range lines[1:] {
497 line = strings.TrimSpace(line)
498 if line != "" {
499 digests = append(digests, line)
500 }
501 }
502 return id, digests
503}
504
505func dockerImageLines(image, id string, digests []string) []string {
506 image = strings.TrimSpace(image)
507 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
508 id = strings.TrimSpace(id)
509 digest := matchingDigest(image, digests)
510 switch {
511 case id != "" && digest != "":
512 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
513 case digest != "":
514 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
515 case id != "":
516 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
517 }
518 return lines
519}
520
521func matchingDigest(image string, digests []string) string {
522 name := imageName(image)
523 fallback := ""
524 for _, digest := range digests {
525 digest = strings.TrimSpace(digest)
526 at := strings.LastIndex(digest, "@")
527 if digest == "" || at <= 0 {
528 continue
529 }
530 if fallback == "" {
531 fallback = digest
532 }
533 repo := digest[:at]
534 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
535 return digest
536 }
537 }
538 return fallback
539}
540
541func imageName(image string) string {
542 image = strings.TrimSpace(image)
543 if i := strings.LastIndex(image, "@"); i >= 0 {
544 image = image[:i]
545 }
546 slash := strings.LastIndex(image, "/")
547 name := image
548 if slash >= 0 {
549 name = image[slash+1:]
550 }
551 if i := strings.Index(name, ":"); i >= 0 {
552 name = name[:i]
553 }
554 return name
555}
556
557func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
558 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
559 if err != nil {
560 return err
561 }
562 wanted := map[string]bool{}
563 if job.NeedsSet {
564 for _, n := range job.Needs {
565 wanted[n] = true
566 }
567 } else {
568 stageOrder := map[string]int{}
569 n := 0
570 for _, j := range jobs {
571 if _, ok := stageOrder[j.Stage]; !ok {
572 stageOrder[j.Stage] = n
573 n++
574 }
575 }
576 my := stageOrder[job.Stage]
577 for _, j := range jobs {
578 if stageOrder[j.Stage] < my {
579 wanted[j.Name] = true
580 }
581 }
582 }
583 for _, j := range jobs {
584 if !wanted[j.Name] || j.Status != "success" {
585 continue
586 }
587 for _, rel := range j.ArtifactPaths {
588 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
589 if err != nil {
590 continue
591 }
592 dst := filepath.Join(work, filepath.FromSlash(rel))
593 if err := copyFile(src, dst); err != nil {
594 return err
595 }
596 }
597 }
598 return nil
599}
600
601func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
602 var expires *time.Time
603 if job.ArtifactExpire > 0 {
604 t := time.Now().Add(job.ArtifactExpire)
605 expires = &t
606 }
607 for _, rel := range job.ArtifactPaths {
608 src := filepath.Join(work, filepath.FromSlash(rel))
609 info, err := os.Stat(src)
610 if err != nil || info.IsDir() {
611 return fmt.Errorf("artifact %s ontbreekt", rel)
612 }
613 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
614 if err := copyFile(src, dst); err != nil {
615 return err
616 }
617 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
618 return err
619 }
620 }
621 return nil
622}
623
624func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
625 name, _ := ci.SplitRef(p.Ref)
626 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
627 if err != nil {
628 return nil, nil, err
629 }
630 refProtected := access.BranchProtected(patterns, name)
631 env := map[string]string{}
632 var mask []string
633 if repo.OwnerKind == "group" {
634 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
635 if err != nil {
636 return nil, nil, err
637 }
638 for k, v := range groupSecrets {
639 env[k] = v
640 mask = append(mask, v)
641 }
642 }
643 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
644 if err != nil {
645 return nil, nil, err
646 }
647 for k, v := range repoSecrets {
648 env[k] = v
649 mask = append(mask, v)
650 }
651 for k, v := range job.Variables {
652 env[k] = v
653 }
654 base := strings.TrimRight(r.BaseURL, "/")
655 var info *ci.MRInfo
656 if p.MRID != 0 {
657 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
658 target := ""
659 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
660 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
661 target = strings.TrimSpace(string(out))
662 }
663 info = &ci.MRInfo{
664 ID: m.ID, IID: m.Number, Title: m.Title,
665 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
666 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
667 TargetSHA: target,
668 }
669 }
670 }
671 pre := ci.Variables(ci.Ref{
672 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle,
673 DefaultBranch: repo.DefaultBranch, Source: p.Source,
674 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
675 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
676 })
677 for k, v := range pre {
678 env[k] = v
679 }
680 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
681 env["CI_JOB_NAME"] = job.Name
682 env["CI_JOB_STAGE"] = job.Stage
683 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
684 env["CI_JOB_TOKEN"] = token
685 if job.Environment != "" {
686 env["CI_ENVIRONMENT_NAME"] = job.Environment
687 if job.EnvironmentURL != "" {
688 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
689 }
690 }
691 for k, v := range job.Variables {
692 if strings.HasPrefix(k, "CI_NODE_") {
693 env[k] = v
694 }
695 }
696 mask = append(mask, token)
697 return env, mask, nil
698}
699
700func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
701 netName := fmt.Sprintf("bluegit-%d", job.ID)
702 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
703 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
704 }
705 var names []string
706 cleanup := func() {
707 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
708 defer cancel()
709 for _, name := range names {
710 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
711 }
712 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
713 }
714 for i, svc := range job.Services {
715 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
716 args := []string{"run", "-d", "--name", cname, "--network", netName}
717 if svc.Alias != "" {
718 args = append(args, "--network-alias", svc.Alias)
719 }
720 cmdEnv := append([]string{}, os.Environ()...)
721 for k, v := range svc.Variables {
722 args = append(args, "-e", k)
723 cmdEnv = append(cmdEnv, k+"="+v)
724 }
725 if len(svc.Entrypoint) > 0 {
726 args = append(args, "--entrypoint", svc.Entrypoint[0])
727 }
728 args = append(args, svc.Name)
729 if len(svc.Entrypoint) > 1 {
730 args = append(args, svc.Entrypoint[1:]...)
731 }
732 args = append(args, svc.Command...)
733 cmd := exec.CommandContext(ctx, r.Docker, args...)
734 cmd.Env = cmdEnv
735 var stderr bytes.Buffer
736 cmd.Stderr = &stderr
737 if err := cmd.Run(); err != nil {
738 cleanup()
739 msg := strings.TrimSpace(stderr.String())
740 if msg == "" {
741 msg = err.Error()
742 }
743 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
744 }
745 names = append(names, cname)
746 }
747 return netName, cleanup, nil
748}
749
750func shouldRetry(job store.Job, kind string) bool {
751 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
752 return false
753 }
754 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
755 return true
756 }
757 if containsStr(job.RetryWhen, kind) {
758 return true
759 }
760 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
761}
762
763func classifyFailure(err error) string {
764 msg := err.Error()
765 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
766 return "runner_system_failure"
767 }
768 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
769 return "job_execution_timeout"
770 }
771 return "script_failure"
772}
773
774func containsStr(list []string, s string) bool {
775 for _, item := range list {
776 if item == s {
777 return true
778 }
779 }
780 return false
781}
782
783func safeSegment(name string) string {
784 var b strings.Builder
785 for _, r := range name {
786 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
787 b.WriteRune(r)
788 } else {
789 b.WriteByte('_')
790 }
791 }
792 if b.Len() == 0 {
793 return "job"
794 }
795 return b.String()
796}
797
798func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
799 if len(job.CachePaths) == 0 {
800 return nil
801 }
802 root := r.cacheDir(repoID, job.CacheKey)
803 for _, rel := range job.CachePaths {
804 src := filepath.Join(root, filepath.FromSlash(rel))
805 if _, err := os.Stat(src); err != nil {
806 continue
807 }
808 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
809 return err
810 }
811 }
812 return nil
813}
814
815func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
816 if len(job.CachePaths) == 0 {
817 return nil
818 }
819 root := r.cacheDir(repoID, job.CacheKey)
820 for _, rel := range job.CachePaths {
821 src := filepath.Join(work, filepath.FromSlash(rel))
822 info, err := os.Stat(src)
823 if err != nil || info.IsDir() {
824 continue
825 }
826 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
827 return err
828 }
829 }
830 return nil
831}
832
833func (r *Runner) cacheDir(repoID int64, key string) string {
834 if key == "" {
835 key = "default"
836 }
837 var b strings.Builder
838 for _, r := range key {
839 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
840 b.WriteRune(r)
841 }
842 }
843 safe := b.String()
844 if safe == "" {
845 safe = "default"
846 }
847 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
848}
849
850func stamp(line string) string {
851 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
852}
853
854type logSink struct {
855 mu sync.Mutex
856 store *store.Store
857 jobID int64
858 secrets []string
859 partial string
860 current string
861 opened time.Time
862}
863
864func (l *logSink) Write(p []byte) (int, error) {
865 l.mu.Lock()
866 defer l.mu.Unlock()
867 l.partial += string(p)
868 for {
869 i := strings.IndexByte(l.partial, '\n')
870 if i < 0 {
871 break
872 }
873 l.emit(strings.TrimRight(l.partial[:i], "\r"))
874 l.partial = l.partial[i+1:]
875 }
876 return len(p), nil
877}
878
879func (l *logSink) flush() {
880 l.mu.Lock()
881 defer l.mu.Unlock()
882 if strings.TrimSpace(l.partial) != "" {
883 l.emit(l.partial)
884 }
885 l.partial = ""
886}
887
888func (l *logSink) emit(line string) {
889 line = ci.Mask(line, l.secrets)
890 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
891 if l.current != "" && strings.HasPrefix(line, "section_end:") {
892 dur := int(time.Since(l.opened).Seconds())
893 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
894 l.current = ""
895 }
896 if strings.HasPrefix(line, "section_start:") {
897 l.current = line
898 l.opened = time.Now()
899 }
900 }
901 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
902}
903
904func (l *logSink) section(name, title string) {
905 l.emit("section_start:" + name + ":" + title)
906}
907
908func (l *logSink) closeSection() {
909 if l.current == "" {
910 return
911 }
912 name := "prepare"
913 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
914 name = parts[0]
915 }
916 l.emit("section_end:" + name + ":done")
917}
918
919func copyFile(src, dst string) error {
920 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
921 return err
922 }
923 in, err := os.Open(src)
924 if err != nil {
925 return err
926 }
927 defer in.Close()
928 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
929 if err != nil {
930 return err
931 }
932 defer out.Close()
933 _, err = io.Copy(out, in)
934 return err
935}
936
937func execOut(ctx context.Context, bin string, args ...string) (string, error) {
938 if bin == "" {
939 bin = "git"
940 }
941 cmd := exec.CommandContext(ctx, bin, args...)
942 var stdout, stderr bytes.Buffer
943 cmd.Stdout = &stdout
944 cmd.Stderr = &stderr
945 if err := cmd.Run(); err != nil {
946 msg := strings.TrimSpace(stderr.String())
947 if msg == "" {
948 msg = err.Error()
949 }
950 return "", fmt.Errorf("%s", msg)
951 }
952 return stdout.String(), nil
953}