BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

feat: breid rules en de ingebouwde runner uit
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
5fec96f0 History
runner.go 22.0 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "strings"
15 "sync"
16 "time"
17
18 "gitlab.com/Timfa/bluegit/internal/access"
19 "gitlab.com/Timfa/bluegit/internal/ci"
20 "gitlab.com/Timfa/bluegit/internal/gitrepo"
21 "gitlab.com/Timfa/bluegit/internal/store"
22 "gitlab.com/Timfa/bluegit/internal/webhook"
23)
24
25// Runner plant jobs en stuurt het resultaat door naar webhooks.
26type Runner struct {
27 Store *store.Store
28 GitBin string
29 Docker string
30 Storage string
31 BaseURL string
32 Network string
33 Timeout time.Duration
34 Log *slog.Logger
35 OnEvent func(repoID int64, ev webhook.Event)
36 slots chan struct{}
37 mu sync.RWMutex
38 cancelMu sync.Mutex
39 cancel map[int64]context.CancelFunc
40}
41
42// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
43func (r *Runner) Interrupt(ids []int64) {
44 r.cancelMu.Lock()
45 defer r.cancelMu.Unlock()
46 for _, id := range ids {
47 if c := r.cancel[id]; c != nil {
48 c()
49 }
50 }
51}
52
53func (r *Runner) track(id int64, cancel context.CancelFunc) {
54 r.cancelMu.Lock()
55 defer r.cancelMu.Unlock()
56 if r.cancel == nil {
57 r.cancel = map[int64]context.CancelFunc{}
58 }
59 r.cancel[id] = cancel
60}
61
62func (r *Runner) untrack(id int64) {
63 r.cancelMu.Lock()
64 defer r.cancelMu.Unlock()
65 delete(r.cancel, id)
66}
67
68// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
69func (r *Runner) SetStorage(path string) {
70 r.mu.Lock()
71 r.Storage = path
72 r.mu.Unlock()
73}
74
75func (r *Runner) storage() string {
76 r.mu.RLock()
77 defer r.mu.RUnlock()
78 return r.Storage
79}
80
81// Loop blijft jobs oppakken tot ctx afloopt.
82func (r *Runner) Loop(ctx context.Context, concurrency int) {
83 if concurrency < 1 {
84 concurrency = 1
85 }
86 r.slots = make(chan struct{}, concurrency)
87 tick := time.NewTicker(2 * time.Second)
88 defer tick.Stop()
89 for {
90 select {
91 case <-ctx.Done():
92 return
93 case <-tick.C:
94 r.Tick(ctx)
95 }
96 }
97}
98
99// Tick bekijkt alle open pipelines één keer.
100// Zonder opslagpad gebeurt er niets.
101func (r *Runner) Tick(ctx context.Context) {
102 if r.storage() == "" {
103 return
104 }
105 pipes, err := r.Store.ActivePipelines(ctx)
106 if err != nil {
107 r.Log.Error("pipelines", "err", err)
108 return
109 }
110 for _, p := range pipes {
111 r.schedule(ctx, p)
112 }
113}
114
115func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
116 r.settle(ctx, p.ID)
117 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
118 if err != nil {
119 r.Log.Error("jobs", "err", err)
120 return
121 }
122 specs := make([]ci.Job, len(jobs))
123 status := map[string]string{}
124 for i, j := range jobs {
125 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
126 status[j.Name] = j.Status
127 }
128 for _, j := range jobs {
129 if j.Status != "queued" && j.Status != "manual" {
130 continue
131 }
132 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
133 case "skip":
134 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
135 status[j.Name] = "skipped"
136 case "fail":
137 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
138 status[j.Name] = "failed"
139 case "ready":
140 if j.Status != "queued" {
141 continue
142 }
143 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
144 continue
145 }
146 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
147 if err != nil || !ok {
148 continue
149 }
150 status[j.Name] = "running"
151 job := j
152 pipe := p
153 go r.execute(ctx, pipe, job)
154 }
155 }
156}
157
158// settle slaat jobs over die door een harde mislukking niet meer hoeven.
159// Dat gebeurt meteen, ook als de pipelinestatus al op failed was gezet.
160func (r *Runner) settle(ctx context.Context, pipelineID int64) {
161 for {
162 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
163 if err != nil {
164 if r.Log != nil {
165 r.Log.Error("jobs", "err", err)
166 }
167 return
168 }
169 specs := make([]ci.Job, len(jobs))
170 status := map[string]string{}
171 for i, j := range jobs {
172 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
173 status[j.Name] = j.Status
174 }
175 moved := false
176 for _, j := range jobs {
177 if j.Status != "queued" && j.Status != "manual" {
178 continue
179 }
180 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
181 case "skip":
182 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
183 r.Log.Error("skip", "err", err)
184 continue
185 }
186 status[j.Name] = "skipped"
187 moved = true
188 case "fail":
189 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
190 r.Log.Error("fail", "err", err)
191 continue
192 }
193 status[j.Name] = "failed"
194 moved = true
195 }
196 }
197 if !moved {
198 return
199 }
200 }
201}
202
203func indexOf(jobs []store.Job, name string) int {
204 for i, j := range jobs {
205 if j.Name == name {
206 return i
207 }
208 }
209 return 0
210}
211
212func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
213 if r.slots != nil {
214 r.slots <- struct{}{}
215 defer func() { <-r.slots }()
216 }
217 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
218 base, stop := context.WithCancel(ctx)
219 r.track(job.ID, stop)
220 defer func() {
221 r.untrack(job.ID)
222 stop()
223 }()
224 runCtx := base
225 limit := r.Timeout
226 if job.Timeout > 0 {
227 limit = job.Timeout
228 }
229 if limit > 0 {
230 var cancel context.CancelFunc
231 runCtx, cancel = context.WithTimeout(base, limit)
232 defer cancel()
233 }
234 mask, err := r.runOne(runCtx, p, job)
235 status := "success"
236 extra := ""
237 kind := ""
238 switch {
239 case err == nil:
240 case ctx.Err() == nil && base.Err() != nil:
241 status = "canceled"
242 extra = stamp("canceled because a newer pipeline started for the same ref")
243 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
244 status = "failed"
245 kind = "job_execution_timeout"
246 extra = stamp(ci.Mask(err.Error(), mask))
247 default:
248 status = "failed"
249 kind = classifyFailure(err)
250 extra = stamp(ci.Mask(err.Error(), mask))
251 }
252 if status == "failed" && shouldRetry(job, kind) {
253 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
254 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
255 r.Log.Error("job opnieuw", "err", ferr)
256 } else if ferr == nil {
257 r.settle(context.Background(), p.ID)
258 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
259 r.Log.Error("pipelinestatus", "err", rerr)
260 }
261 return
262 }
263 }
264 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
265 r.Log.Error("job afronden", "err", ferr)
266 }
267 r.settle(context.Background(), p.ID)
268 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
269 if rerr != nil {
270 r.Log.Error("pipelinestatus", "err", rerr)
271 return
272 }
273 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
274 kind := webhook.EventPipelineSuccess
275 word := "geslaagd"
276 if final == "failed" {
277 kind = webhook.EventPipelineFailure
278 word = "mislukt"
279 }
280 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
281 if err != nil {
282 return
283 }
284 branch, _ := gitrepo.BranchOfRef(p.Ref)
285 if branch == "" {
286 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
287 }
288 r.OnEvent(p.RepoID, webhook.Event{
289 Type: kind,
290 Project: repo.FullPath(),
291 Branch: branch,
292 Commit: p.CommitSHA,
293 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
294 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
295 PipelineID: fmt.Sprint(p.ID),
296 })
297 }
298}
299
300func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
301 repo, err := r.Store.RepoByID(ctx, p.RepoID)
302 if err != nil {
303 return nil, err
304 }
305 sink := &logSink{store: r.Store, jobID: job.ID}
306 sink.section("prepare", "Preparing the docker executor")
307 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
308 _ = os.RemoveAll(work)
309 if err := os.MkdirAll(work, 0o755); err != nil {
310 sink.closeSection()
311 return nil, err
312 }
313 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
314 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
315 sink.closeSection()
316 return nil, fmt.Errorf("clone: %w", err)
317 }
318 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
319 sink.closeSection()
320 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
321 }
322 if err := r.restoreCache(repo.ID, job, work); err != nil {
323 sink.closeSection()
324 return nil, err
325 }
326 if err := r.materialize(ctx, p, job, work); err != nil {
327 sink.closeSection()
328 return nil, err
329 }
330 scriptPath := filepath.Join(work, ".bluegit-job.sh")
331 if err := os.WriteFile(scriptPath, []byte(shellScript(job)), 0o755); err != nil {
332 sink.closeSection()
333 return nil, err
334 }
335 token, err := r.Store.CreateJobToken(ctx, job.ID)
336 if err != nil {
337 sink.closeSection()
338 return nil, err
339 }
340 env, mask, err := r.environment(ctx, repo, p, job, token)
341 if err != nil {
342 sink.closeSection()
343 return nil, err
344 }
345 sink.secrets = mask
346 if _, err := exec.LookPath(r.Docker); err != nil {
347 sink.closeSection()
348 return mask, fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
349 }
350 network := r.Network
351 if len(job.Services) > 0 {
352 net, cleanup, serr := r.startServices(ctx, job)
353 if serr != nil {
354 sink.closeSection()
355 return mask, serr
356 }
357 defer cleanup()
358 network = net
359 }
360 sink.closeSection()
361 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
362 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
363 // staat de waarde niet in de proceslijst.
364 args := []string{"run", "--rm", "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
365 cmdEnv := append([]string{}, os.Environ()...)
366 for k, v := range env {
367 args = append(args, "-e", k)
368 cmdEnv = append(cmdEnv, k+"="+v)
369 }
370 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
371 cmd := exec.CommandContext(ctx, r.Docker, args...)
372 cmd.Env = cmdEnv
373 cmd.Stdout = sink
374 cmd.Stderr = sink
375 err = cmd.Run()
376 sink.flush()
377 if err != nil {
378 return mask, fmt.Errorf("container ended with an error: %w", err)
379 }
380 if err := r.collect(ctx, job, work); err != nil {
381 return mask, err
382 }
383 if err := r.saveCache(repo.ID, job, work); err != nil {
384 return mask, err
385 }
386 return mask, nil
387}
388
389func shellScript(job store.Job) string {
390 var b strings.Builder
391 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
392 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
393 if strings.TrimSpace(job.Before) != "" {
394 b.WriteString("(\nset -e\n")
395 b.WriteString(job.Before)
396 if !strings.HasSuffix(job.Before, "\n") {
397 b.WriteByte('\n')
398 }
399 b.WriteString(")\nbrc=$?\n")
400 } else {
401 b.WriteString("brc=0\n")
402 }
403 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
404 b.WriteString("echo \"section_start:script:Running script\"\n")
405 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
406 b.WriteString(job.Script)
407 if !strings.HasSuffix(job.Script, "\n") {
408 b.WriteByte('\n')
409 }
410 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
411 b.WriteString("echo \"section_end:script:Running script\"\n")
412 b.WriteString("echo \"section_start:after:After script\"\n")
413 if strings.TrimSpace(job.After) != "" {
414 b.WriteString("(\nset -e\n")
415 b.WriteString(job.After)
416 if !strings.HasSuffix(job.After, "\n") {
417 b.WriteByte('\n')
418 }
419 b.WriteString(")\n")
420 }
421 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
422 return b.String()
423}
424
425func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
426 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
427 if err != nil {
428 return err
429 }
430 wanted := map[string]bool{}
431 if job.NeedsSet {
432 for _, n := range job.Needs {
433 wanted[n] = true
434 }
435 } else {
436 stageOrder := map[string]int{}
437 n := 0
438 for _, j := range jobs {
439 if _, ok := stageOrder[j.Stage]; !ok {
440 stageOrder[j.Stage] = n
441 n++
442 }
443 }
444 my := stageOrder[job.Stage]
445 for _, j := range jobs {
446 if stageOrder[j.Stage] < my {
447 wanted[j.Name] = true
448 }
449 }
450 }
451 for _, j := range jobs {
452 if !wanted[j.Name] || j.Status != "success" {
453 continue
454 }
455 for _, rel := range j.ArtifactPaths {
456 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
457 if err != nil {
458 continue
459 }
460 dst := filepath.Join(work, filepath.FromSlash(rel))
461 if err := copyFile(src, dst); err != nil {
462 return err
463 }
464 }
465 }
466 return nil
467}
468
469func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
470 var expires *time.Time
471 if job.ArtifactExpire > 0 {
472 t := time.Now().Add(job.ArtifactExpire)
473 expires = &t
474 }
475 for _, rel := range job.ArtifactPaths {
476 src := filepath.Join(work, filepath.FromSlash(rel))
477 info, err := os.Stat(src)
478 if err != nil || info.IsDir() {
479 return fmt.Errorf("artifact %s ontbreekt", rel)
480 }
481 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
482 if err := copyFile(src, dst); err != nil {
483 return err
484 }
485 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
486 return err
487 }
488 }
489 return nil
490}
491
492func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
493 name, _ := ci.SplitRef(p.Ref)
494 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
495 if err != nil {
496 return nil, nil, err
497 }
498 refProtected := access.BranchProtected(patterns, name)
499 env := map[string]string{}
500 var mask []string
501 if repo.OwnerKind == "group" {
502 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
503 if err != nil {
504 return nil, nil, err
505 }
506 for k, v := range groupSecrets {
507 env[k] = v
508 mask = append(mask, v)
509 }
510 }
511 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
512 if err != nil {
513 return nil, nil, err
514 }
515 for k, v := range repoSecrets {
516 env[k] = v
517 mask = append(mask, v)
518 }
519 for k, v := range job.Variables {
520 env[k] = v
521 }
522 base := strings.TrimRight(r.BaseURL, "/")
523 var info *ci.MRInfo
524 if p.MRID != 0 {
525 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
526 target := ""
527 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
528 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
529 target = strings.TrimSpace(string(out))
530 }
531 info = &ci.MRInfo{
532 ID: m.ID, IID: m.Number, Title: m.Title,
533 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
534 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
535 TargetSHA: target,
536 }
537 }
538 }
539 pre := ci.Variables(ci.Ref{
540 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle,
541 DefaultBranch: repo.DefaultBranch, Source: p.Source,
542 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
543 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
544 })
545 for k, v := range pre {
546 env[k] = v
547 }
548 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
549 env["CI_JOB_NAME"] = job.Name
550 env["CI_JOB_STAGE"] = job.Stage
551 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
552 env["CI_JOB_TOKEN"] = token
553 if job.Environment != "" {
554 env["CI_ENVIRONMENT_NAME"] = job.Environment
555 if job.EnvironmentURL != "" {
556 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
557 }
558 }
559 for k, v := range job.Variables {
560 if strings.HasPrefix(k, "CI_NODE_") {
561 env[k] = v
562 }
563 }
564 mask = append(mask, token)
565 return env, mask, nil
566}
567
568func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
569 netName := fmt.Sprintf("bluegit-%d", job.ID)
570 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
571 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
572 }
573 var names []string
574 cleanup := func() {
575 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
576 defer cancel()
577 for _, name := range names {
578 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
579 }
580 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
581 }
582 for i, svc := range job.Services {
583 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
584 args := []string{"run", "-d", "--name", cname, "--network", netName}
585 if svc.Alias != "" {
586 args = append(args, "--network-alias", svc.Alias)
587 }
588 cmdEnv := append([]string{}, os.Environ()...)
589 for k, v := range svc.Variables {
590 args = append(args, "-e", k)
591 cmdEnv = append(cmdEnv, k+"="+v)
592 }
593 if len(svc.Entrypoint) > 0 {
594 args = append(args, "--entrypoint", svc.Entrypoint[0])
595 }
596 args = append(args, svc.Name)
597 if len(svc.Entrypoint) > 1 {
598 args = append(args, svc.Entrypoint[1:]...)
599 }
600 args = append(args, svc.Command...)
601 cmd := exec.CommandContext(ctx, r.Docker, args...)
602 cmd.Env = cmdEnv
603 var stderr bytes.Buffer
604 cmd.Stderr = &stderr
605 if err := cmd.Run(); err != nil {
606 cleanup()
607 msg := strings.TrimSpace(stderr.String())
608 if msg == "" {
609 msg = err.Error()
610 }
611 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
612 }
613 names = append(names, cname)
614 }
615 return netName, cleanup, nil
616}
617
618func shouldRetry(job store.Job, kind string) bool {
619 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
620 return false
621 }
622 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
623 return true
624 }
625 if containsStr(job.RetryWhen, kind) {
626 return true
627 }
628 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
629}
630
631func classifyFailure(err error) string {
632 msg := err.Error()
633 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
634 return "runner_system_failure"
635 }
636 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
637 return "job_execution_timeout"
638 }
639 return "script_failure"
640}
641
642func containsStr(list []string, s string) bool {
643 for _, item := range list {
644 if item == s {
645 return true
646 }
647 }
648 return false
649}
650
651func safeSegment(name string) string {
652 var b strings.Builder
653 for _, r := range name {
654 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
655 b.WriteRune(r)
656 } else {
657 b.WriteByte('_')
658 }
659 }
660 if b.Len() == 0 {
661 return "job"
662 }
663 return b.String()
664}
665
666func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
667 if len(job.CachePaths) == 0 {
668 return nil
669 }
670 root := r.cacheDir(repoID, job.CacheKey)
671 for _, rel := range job.CachePaths {
672 src := filepath.Join(root, filepath.FromSlash(rel))
673 if _, err := os.Stat(src); err != nil {
674 continue
675 }
676 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
677 return err
678 }
679 }
680 return nil
681}
682
683func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
684 if len(job.CachePaths) == 0 {
685 return nil
686 }
687 root := r.cacheDir(repoID, job.CacheKey)
688 for _, rel := range job.CachePaths {
689 src := filepath.Join(work, filepath.FromSlash(rel))
690 info, err := os.Stat(src)
691 if err != nil || info.IsDir() {
692 continue
693 }
694 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
695 return err
696 }
697 }
698 return nil
699}
700
701func (r *Runner) cacheDir(repoID int64, key string) string {
702 if key == "" {
703 key = "default"
704 }
705 var b strings.Builder
706 for _, r := range key {
707 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
708 b.WriteRune(r)
709 }
710 }
711 safe := b.String()
712 if safe == "" {
713 safe = "default"
714 }
715 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
716}
717
718func stamp(line string) string {
719 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
720}
721
722type logSink struct {
723 mu sync.Mutex
724 store *store.Store
725 jobID int64
726 secrets []string
727 partial string
728 current string
729 opened time.Time
730}
731
732func (l *logSink) Write(p []byte) (int, error) {
733 l.mu.Lock()
734 defer l.mu.Unlock()
735 l.partial += string(p)
736 for {
737 i := strings.IndexByte(l.partial, '\n')
738 if i < 0 {
739 break
740 }
741 l.emit(strings.TrimRight(l.partial[:i], "\r"))
742 l.partial = l.partial[i+1:]
743 }
744 return len(p), nil
745}
746
747func (l *logSink) flush() {
748 l.mu.Lock()
749 defer l.mu.Unlock()
750 if strings.TrimSpace(l.partial) != "" {
751 l.emit(l.partial)
752 }
753 l.partial = ""
754}
755
756func (l *logSink) emit(line string) {
757 line = ci.Mask(line, l.secrets)
758 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
759 if l.current != "" && strings.HasPrefix(line, "section_end:") {
760 dur := int(time.Since(l.opened).Seconds())
761 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
762 l.current = ""
763 }
764 if strings.HasPrefix(line, "section_start:") {
765 l.current = line
766 l.opened = time.Now()
767 }
768 }
769 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
770}
771
772func (l *logSink) section(name, title string) {
773 l.emit("section_start:" + name + ":" + title)
774}
775
776func (l *logSink) closeSection() {
777 if l.current == "" {
778 return
779 }
780 name := "prepare"
781 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
782 name = parts[0]
783 }
784 l.emit("section_end:" + name + ":done")
785}
786
787func copyFile(src, dst string) error {
788 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
789 return err
790 }
791 in, err := os.Open(src)
792 if err != nil {
793 return err
794 }
795 defer in.Close()
796 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
797 if err != nil {
798 return err
799 }
800 defer out.Close()
801 _, err = io.Copy(out, in)
802 return err
803}
804
805func execOut(ctx context.Context, bin string, args ...string) (string, error) {
806 if bin == "" {
807 bin = "git"
808 }
809 cmd := exec.CommandContext(ctx, bin, args...)
810 var stdout, stderr bytes.Buffer
811 cmd.Stdout = &stdout
812 cmd.Stderr = &stderr
813 if err := cmd.Run(); err != nil {
814 msg := strings.TrimSpace(stderr.String())
815 if msg == "" {
816 msg = err.Error()
817 }
818 return "", fmt.Errorf("%s", msg)
819 }
820 return stdout.String(), nil
821}