BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 39.6 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "os"
13 "os/exec"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "sync"
18 "sync/atomic"
19 "time"
20
21 "gitlab.com/Timfa/bluegit/internal/access"
22 "gitlab.com/Timfa/bluegit/internal/ci"
23 "gitlab.com/Timfa/bluegit/internal/gitrepo"
24 "gitlab.com/Timfa/bluegit/internal/images"
25 "gitlab.com/Timfa/bluegit/internal/runnercfg"
26 "gitlab.com/Timfa/bluegit/internal/store"
27 "gitlab.com/Timfa/bluegit/internal/webhook"
28)
29
30// Runner plant jobs en stuurt het resultaat door naar webhooks.
31type Runner struct {
32 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
33 Hold func(store.Pipeline) bool
34 Store *store.Store
35 GitBin string
36 Docker string
37 Storage string
38 BaseURL string
39 Network string
40 Name string
41 Timeout time.Duration
42 Log *slog.Logger
43 OnEvent func(repoID int64, ev webhook.Event)
44 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
45 Presets map[string]images.Override
46 // Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
47 Jobs *runnercfg.Tracker
48 ctl control
49 slots chan struct{}
50 mu sync.RWMutex
51 cancelMu sync.Mutex
52 // protected telt lopende doNotInterrupt-jobs van de ingebouwde runner.
53 protected atomic.Int64
54 cancel map[int64]context.CancelFunc
55}
56
57// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
58func (r *Runner) Interrupt(ids []int64) {
59 r.cancelMu.Lock()
60 defer r.cancelMu.Unlock()
61 for _, id := range ids {
62 if c := r.cancel[id]; c != nil {
63 c()
64 }
65 }
66}
67
68// cancelReason leest waarom een lopende job gestopt werd. CancelJob (door
69// een gebruiker) zet canceled_by_user al in de database; anders was het
70// een nieuwere pipeline op dezelfde ref.
71func (r *Runner) cancelReason(repoID, id int64) string {
72 if j, err := r.Store.JobByID(context.Background(), repoID, id); err == nil && j.Reason == "canceled_by_user" {
73 return j.Reason
74 }
75 return "canceled_by_newer_pipeline"
76}
77
78func cancelText(reason string) string {
79 if reason == "canceled_by_user" {
80 return "canceled by a user"
81 }
82 return "canceled because a newer pipeline started for the same ref"
83}
84
85func (r *Runner) track(id int64, cancel context.CancelFunc) {
86 r.cancelMu.Lock()
87 defer r.cancelMu.Unlock()
88 if r.cancel == nil {
89 r.cancel = map[int64]context.CancelFunc{}
90 }
91 r.cancel[id] = cancel
92}
93
94func (r *Runner) untrack(id int64) {
95 r.cancelMu.Lock()
96 defer r.cancelMu.Unlock()
97 delete(r.cancel, id)
98}
99
100// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
101func (r *Runner) SetStorage(path string) {
102 r.mu.Lock()
103 r.Storage = path
104 r.mu.Unlock()
105}
106
107func (r *Runner) storage() string {
108 r.mu.RLock()
109 defer r.mu.RUnlock()
110 return r.Storage
111}
112
113// Loop blijft jobs oppakken tot ctx afloopt.
114func (r *Runner) Loop(ctx context.Context, concurrency int) {
115 if concurrency < 1 {
116 concurrency = 1
117 }
118 r.slots = make(chan struct{}, maxSlots)
119 r.mu.Lock()
120 if !r.ctl.set {
121 r.ctl.limit = concurrency
122 }
123 r.mu.Unlock()
124 tick := time.NewTicker(2 * time.Second)
125 defer tick.Stop()
126 for {
127 select {
128 case <-ctx.Done():
129 return
130 case <-tick.C:
131 r.Tick(ctx)
132 }
133 }
134}
135
136// Tick bekijkt alle open pipelines één keer.
137// Zonder opslagpad gebeurt er niets.
138func (r *Runner) Tick(ctx context.Context) {
139 if r.storage() == "" {
140 return
141 }
142 pipes, err := r.Store.ActivePipelines(ctx)
143 if err != nil {
144 r.Log.Error("pipelines", "err", err)
145 return
146 }
147 for _, p := range pipes {
148 r.schedule(ctx, p)
149 }
150 r.reapRemote(ctx)
151}
152
153func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
154 r.settle(ctx, p.ID)
155 if p.Status == "running" {
156 // Een pipeline die alleen nog op handmatige jobs wacht, is niet
157 // meer bezig (ook oude pipelines van voor deze regel).
158 if st, err := r.Store.RefreshPipelineStatus(ctx, p.ID); err == nil {
159 p.Status = st
160 }
161 }
162 if r.Hold != nil && r.Hold(p) {
163 return // wacht op de self-update
164 }
165 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
166 if err != nil {
167 r.Log.Error("jobs", "err", err)
168 return
169 }
170 specs := make([]ci.Job, len(jobs))
171 status := map[string]string{}
172 for i, j := range jobs {
173 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
174 status[j.Name] = j.Status
175 }
176 tierCache := map[int64][]Tier{}
177 for _, j := range jobs {
178 if j.Status != "queued" && j.Status != "manual" {
179 continue
180 }
181 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
182 case "skip":
183 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
184 status[j.Name] = "skipped"
185 case "fail":
186 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
187 status[j.Name] = "failed"
188 case "ready":
189 if j.Status != "queued" {
190 continue
191 }
192 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
193 continue
194 }
195 // Meer jobs dan ci.concurrency blijven queued, met een plek
196 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
197 free := r.acquireSlot()
198 prot := r.refProtected(ctx, j.RepoID, p.Ref)
199 if prot {
200 r.noteProtected(ctx, j.RepoID, j)
201 }
202 if tiers := r.tiersFor(ctx, tierCache, j.RepoID, prot); len(tiers) > 1 {
203 jr := r.Store.JobRun(ctx, j.ID)
204 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
205 if !step.Builtin {
206 if free {
207 r.releaseSlot()
208 }
209 if step.Moved {
210 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
211 }
212 continue
213 }
214 }
215 if !free {
216 continue
217 }
218 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
219 if err != nil || !ok {
220 r.releaseSlot()
221 continue
222 }
223 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
224 _ = r.Store.ClearJobOffer(ctx, j.ID)
225 status[j.Name] = "running"
226 job := j
227 pipe := p
228 go r.execute(ctx, pipe, job)
229 }
230 }
231}
232
233// settle slaat jobs over die door een harde mislukking of een
234// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
235// pipelinestatus al op failed was gezet.
236func (r *Runner) settle(ctx context.Context, pipelineID int64) {
237 for {
238 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
239 if err != nil {
240 if r.Log != nil {
241 r.Log.Error("jobs", "err", err)
242 }
243 return
244 }
245 specs := make([]ci.Job, len(jobs))
246 status := map[string]string{}
247 for i, j := range jobs {
248 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
249 status[j.Name] = j.Status
250 }
251 moved := false
252 for _, j := range jobs {
253 if j.Status != "queued" && j.Status != "manual" {
254 continue
255 }
256 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
257 case "skip":
258 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
259 r.Log.Error("skip", "err", err)
260 continue
261 }
262 status[j.Name] = "skipped"
263 moved = true
264 case "fail":
265 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
266 r.Log.Error("fail", "err", err)
267 continue
268 }
269 status[j.Name] = "failed"
270 moved = true
271 }
272 }
273 if !moved {
274 return
275 }
276 }
277}
278
279// BuiltinName is de naam van de ingebouwde runner in de job-UI.
280func (r *Runner) BuiltinName() string {
281 if r.Name != "" {
282 return r.Name
283 }
284 return "BlueGit runner"
285}
286
287func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64, protected bool) []Tier {
288 key := repoID * 2
289 if protected {
290 key++
291 }
292 if t, ok := cache[key]; ok {
293 return t
294 }
295 var tiers []Tier
296 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
297 tiers, _ = Tiers(ctx, r.Store, repo, protected)
298 }
299 cache[key] = tiers
300 return tiers
301}
302
303// refProtected: draait een pipeline op een beschermde ref (en krijgt
304// hij dus beschermde geheimen)?
305func (r *Runner) refProtected(ctx context.Context, repoID int64, ref string) bool {
306 prot, err := r.refIsProtected(ctx, repoID, ref)
307 if err != nil {
308 return true // bij twijfel streng
309 }
310 return prot
311}
312
313// refIsProtected: een branch is beschermd als hij op een beschermd
314// branchpatroon past, een tag als hij op een beschermd tagpatroon past
315// (alleen maintainers mogen zo'n tag pushen). Een tag telt nooit via een
316// branchpatroon, een branch nooit via een tagpatroon.
317func (r *Runner) refIsProtected(ctx context.Context, repoID int64, ref string) (bool, error) {
318 name, kind := ci.SplitRef(ref)
319 if kind == "tag" {
320 patterns, err := r.Store.ProtectedTagPatterns(ctx, repoID)
321 if err != nil {
322 return false, err
323 }
324 return access.BranchProtected(patterns, name), nil
325 }
326 patterns, err := r.Store.ProtectedPatterns(ctx, repoID)
327 if err != nil {
328 return false, err
329 }
330 return access.BranchProtected(patterns, name), nil
331}
332
333var protectedNoted sync.Map // job id → melding al in het log
334
335// noteProtected zet één keer een melding in het joblog als er opt-in
336// runners zijn die deze job niet mogen draaien.
337func (r *Runner) noteProtected(ctx context.Context, repoID int64, job store.Job) {
338 if _, done := protectedNoted.LoadOrStore(job.ID, true); done {
339 return
340 }
341 repo, err := r.Store.RepoByID(ctx, repoID)
342 if err != nil {
343 return
344 }
345 if names := ExcludedForProtected(ctx, r.Store, repo); len(names) > 0 {
346 _ = r.Store.AppendJobLog(ctx, job.ID, "Protected branch: this job only runs on the built-in runner, global runners, group runners and account runners of maintainers. Skipped: "+strings.Join(names, ", ")+"\n")
347 }
348}
349
350func indexOf(jobs []store.Job, name string) int {
351 for i, j := range jobs {
352 if j.Name == name {
353 return i
354 }
355 }
356 return 0
357}
358
359func (r *Runner) acquireSlot() bool {
360 if r.Paused() {
361 return false
362 }
363 if r.slots == nil {
364 return true
365 }
366 if l := r.controls().limit; l > 0 && len(r.slots) >= l {
367 return false
368 }
369 select {
370 case r.slots <- struct{}{}:
371 return true
372 default:
373 return false
374 }
375}
376
377func (r *Runner) releaseSlot() {
378 if r.slots == nil {
379 return
380 }
381 select {
382 case <-r.slots:
383 default:
384 }
385}
386
387func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
388 defer r.releaseSlot()
389 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
390 parent := ctx
391 if job.DoNotInterrupt {
392 // Een herstart of stop van BlueGit breekt deze job niet af; main
393 // wacht met afsluiten tot hij klaar is (zie WaitProtected).
394 parent = context.WithoutCancel(ctx)
395 r.protected.Add(1)
396 defer r.protected.Add(-1)
397 }
398 base, stop := context.WithCancel(parent)
399 r.track(job.ID, stop)
400 defer func() {
401 r.untrack(job.ID)
402 stop()
403 }()
404 runCtx := base
405 limit, source := EffectiveTimeout(job.Timeout, r.Timeout, r.controls().timeout)
406 r.trackStart(job)
407 if limit > 0 {
408 var cancel context.CancelFunc
409 runCtx, cancel = context.WithTimeout(base, limit)
410 defer cancel()
411 }
412 mask, err := r.runOne(runCtx, p, job)
413 status := "success"
414 extra := ""
415 kind := ""
416 switch {
417 case err == nil:
418 case parent.Err() == nil && base.Err() != nil:
419 status = "canceled"
420 kind = r.cancelReason(job.RepoID, job.ID)
421 extra = stamp(cancelText(kind))
422 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
423 status = "failed"
424 kind = "job_execution_timeout"
425 extra = stamp(TimedOutLine(limit, source))
426 default:
427 status = "failed"
428 kind = classifyFailure(err)
429 extra = stamp(ci.Mask(err.Error(), mask))
430 }
431 r.trackDone(job.ID, status)
432 r.complete(p, job, before, status, extra, kind)
433}
434
435// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
436// en webhook. Gedeeld door de ingebouwde en de aparte runner.
437func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
438 if status == "failed" && shouldRetry(job, kind) {
439 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
440 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
441 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
442 r.Log.Error("job opnieuw", "err", ferr)
443 } else if ferr == nil {
444 r.settle(context.Background(), p.ID)
445 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
446 r.Log.Error("pipelinestatus", "err", rerr)
447 }
448 return
449 }
450 }
451 if ferr := r.Store.FinishJobReason(context.Background(), job.ID, status, extra, kind); ferr != nil {
452 r.Log.Error("job afronden", "err", ferr)
453 }
454 r.settle(context.Background(), p.ID)
455 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
456 if rerr != nil {
457 r.Log.Error("pipelinestatus", "err", rerr)
458 return
459 }
460 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
461 kind := webhook.EventPipelineSuccess
462 word := "geslaagd"
463 if final == "failed" {
464 kind = webhook.EventPipelineFailure
465 word = "mislukt"
466 }
467 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
468 if err != nil {
469 return
470 }
471 branch, _ := gitrepo.BranchOfRef(p.Ref)
472 if branch == "" {
473 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
474 }
475 r.OnEvent(p.RepoID, webhook.Event{
476 Type: kind,
477 Project: repo.FullPath(),
478 Branch: branch,
479 Commit: p.CommitSHA,
480 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
481 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
482 PipelineID: fmt.Sprint(p.ID),
483 })
484 }
485}
486
487func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
488 repo, err := r.Store.RepoByID(ctx, p.RepoID)
489 if err != nil {
490 return nil, err
491 }
492 sink := &logSink{store: r.Store, jobID: job.ID}
493 sink.section("prepare", "Preparing the docker executor")
494 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
495 if err := clearWorkDir(work); err != nil {
496 sink.closeSection()
497 return nil, fmt.Errorf("runner_system_failure: work dir: %w", err)
498 }
499 if err := os.MkdirAll(work, 0o755); err != nil {
500 sink.closeSection()
501 return nil, err
502 }
503 // Na afloop (ook bij falen of annuleren) gaat de werkmap weg. Artifacts
504 // en cache staan dan al gekopieerd buiten de werkmap (collect, saveCache)
505 // en de container is gestopt. Met cleanup_work uit blijft hij staan.
506 defer r.finishWorkDir(work)
507 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
508 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
509 sink.closeSection()
510 return nil, fmt.Errorf("clone: %w", err)
511 }
512 // origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
513 // op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
514 if r.BaseURL != "" {
515 _, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
516 }
517 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
518 sink.closeSection()
519 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
520 }
521 if err := r.restoreCache(repo.ID, job, work); err != nil {
522 sink.closeSection()
523 return nil, err
524 }
525 if err := r.materialize(ctx, p, job, work); err != nil {
526 sink.closeSection()
527 return nil, err
528 }
529 scriptPath := filepath.Join(work, ".bluegit-job.sh")
530 host, _ := os.Hostname()
531 script := shellScript(job, placeNote{
532 Runner: r.Name, Host: host, WorkDir: "/workspace",
533 PipelineID: p.ID, JobID: job.ID,
534 })
535 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
536 sink.closeSection()
537 return nil, err
538 }
539 token, err := r.Store.CreateJobToken(ctx, job.ID)
540 if err != nil {
541 sink.closeSection()
542 return nil, err
543 }
544 env, mask, err := r.environment(ctx, repo, p, job, token)
545 if err != nil {
546 sink.closeSection()
547 return nil, err
548 }
549 sink.secrets = mask
550 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
551 return mask, err
552 }
553 if err := r.collect(ctx, job, work); err != nil {
554 return mask, err
555 }
556 if err := r.saveCache(repo.ID, job, work); err != nil {
557 return mask, err
558 }
559 return mask, nil
560}
561
562// execContainer haalt het image op, start services en draait het
563// jobscript in de container. De sectie prepare moet open staan.
564func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
565 ref, lines, rerr := r.resolveImage(ctx, job.Image)
566 for _, line := range lines {
567 sink.emit(line)
568 }
569 if rerr != nil {
570 sink.closeSection()
571 return rerr
572 }
573 job.Image = ref
574 job.Services = append([]store.Service(nil), job.Services...)
575 for i := range job.Services {
576 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
577 for _, line := range slines {
578 sink.emit(line)
579 }
580 if serr != nil {
581 sink.closeSection()
582 return serr
583 }
584 job.Services[i].Name = sref
585 }
586 network := r.Network
587 if network == "" {
588 network = "bridge"
589 }
590 if len(job.Services) > 0 {
591 net, cleanup, serr := r.startServices(ctx, job)
592 if serr != nil {
593 sink.closeSection()
594 return serr
595 }
596 defer cleanup()
597 network = net
598 }
599 sink.closeSection()
600 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
601 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
602 // staat de waarde niet in de proceslijst.
603 // Een vaste naam per job: een geannuleerde of afgebroken container
604 // (de docker-client stopt dan, de container niet) wordt voor de start en
605 // na afloop opgeruimd.
606 cname := fmt.Sprintf("bluegit-job-%d", job.ID)
607 removeContainer(r.Docker, cname)
608 defer removeContainer(r.Docker, cname)
609 args := []string{"run", "--rm", "--name", cname, "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
610 ctl := r.controls()
611 args = append(args, ctl.runArgs...)
612 if ctl.pull == runnercfg.PullNever {
613 args = append(args, "--pull", "never")
614 }
615 cmdEnv := append([]string{}, os.Environ()...)
616 for k, v := range env {
617 args = append(args, "-e", k)
618 cmdEnv = append(cmdEnv, k+"="+v)
619 }
620 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
621 cmd := exec.CommandContext(ctx, r.Docker, args...)
622 cmd.Env = cmdEnv
623 cmd.Stdout = sink
624 cmd.Stderr = sink
625 err := cmd.Run()
626 sink.flush()
627 if err != nil {
628 return fmt.Errorf("container ended with an error: %w", err)
629 }
630 return nil
631}
632
633func removeContainer(docker, name string) {
634 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
635 defer cancel()
636 _, _ = execOut(ctx, docker, "rm", "-f", name)
637}
638
639// finishWorkDir ruimt de werkmap van een afgelopen job op, plus de lege
640// pipelinemap erboven, tenzij de instelling cleanup_work uit staat.
641func (r *Runner) finishWorkDir(work string) {
642 if r.controls().keep {
643 return
644 }
645 _ = clearWorkDir(work)
646 _ = os.Remove(filepath.Dir(work)) // alleen als hij leeg is
647}
648
649// ClearWorkDir is clearWorkDir voor de aparte runner.
650func ClearWorkDir(work string) error { return clearWorkDir(work) }
651
652// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
653// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
654// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
655// een lege map begint. Alleen als ook dat faalt, is het een fout.
656func clearWorkDir(work string) error {
657 if err := os.RemoveAll(work); err == nil {
658 return nil
659 }
660 _ = filepath.WalkDir(work, func(p string, d fs.DirEntry, err error) error {
661 if err == nil && d.IsDir() {
662 _ = os.Chmod(p, 0o755)
663 }
664 return nil
665 })
666 if err := os.RemoveAll(work); err == nil {
667 return nil
668 }
669 stale := fmt.Sprintf("%s.stale-%d", work, time.Now().UnixNano())
670 if err := os.Rename(work, stale); err != nil {
671 return err
672 }
673 go func() { _ = os.RemoveAll(stale) }()
674 return nil
675}
676
677// placeNote zijn de vaste regels in de sectie Preparing environment.
678type placeNote struct {
679 Runner string
680 Host string
681 WorkDir string
682 PipelineID int64
683 JobID int64
684}
685
686func (n placeNote) lines() []string {
687 runner := strings.TrimSpace(n.Runner)
688 if runner == "" {
689 runner = "Steve"
690 }
691 host := strings.TrimSpace(n.Host)
692 if host == "" {
693 host = "localhost"
694 }
695 dir := strings.TrimSpace(n.WorkDir)
696 if dir == "" {
697 dir = "/workspace"
698 }
699 return []string{
700 fmt.Sprintf("Running on %s via %s...", runner, host),
701 fmt.Sprintf("Working directory: %s", dir),
702 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
703 }
704}
705
706func shSingle(s string) string {
707 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
708}
709
710func shellScript(job store.Job, note placeNote) string {
711 var b strings.Builder
712 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
713 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
714 for _, line := range note.lines() {
715 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
716 }
717 if strings.TrimSpace(job.Before) != "" {
718 b.WriteString("(\nset -e\n")
719 b.WriteString(job.Before)
720 if !strings.HasSuffix(job.Before, "\n") {
721 b.WriteByte('\n')
722 }
723 b.WriteString(")\nbrc=$?\n")
724 } else {
725 b.WriteString("brc=0\n")
726 }
727 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
728 b.WriteString("echo \"section_start:script:Running script\"\n")
729 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
730 b.WriteString(job.Script)
731 if !strings.HasSuffix(job.Script, "\n") {
732 b.WriteByte('\n')
733 }
734 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
735 b.WriteString("echo \"section_end:script:Running script\"\n")
736 b.WriteString("echo \"section_start:after:After script\"\n")
737 if strings.TrimSpace(job.After) != "" {
738 b.WriteString("(\nset -e\n")
739 b.WriteString(job.After)
740 if !strings.HasSuffix(job.After, "\n") {
741 b.WriteByte('\n')
742 }
743 b.WriteString(")\n")
744 }
745 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
746 return b.String()
747}
748
749// resolveImage kiest het image waarmee de container start.
750// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
751func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
752 image = strings.TrimSpace(image)
753 set, err := images.Effective(r.presetSet())
754 if err != nil {
755 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
756 }
757 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
758 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
759 if p, ok := images.Find(set, image); ok {
760 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
761 }
762 return image, dockerImageLines(image, "", nil), missing
763 }
764 p, ok := images.Find(set, image)
765 if !ok {
766 if r.controls().pull == runnercfg.PullAlways {
767 _, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
768 }
769 id, digests := r.inspectImage(ctx, image)
770 return image, dockerImageLines(image, id, digests), nil
771 }
772 ref, built, err := p.Ensure(ctx, r.Docker)
773 if err != nil {
774 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
775 }
776 var lines []string
777 if built {
778 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
779 }
780 id, digests := r.inspectImage(ctx, ref)
781 var baseDigests []string
782 if ref != p.From {
783 _, baseDigests = r.inspectImage(ctx, p.From)
784 }
785 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
786 return ref, lines, nil
787}
788
789func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
790 var lines []string
791 if preset != "" {
792 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
793 }
794 if ref == "" {
795 return lines
796 }
797 lines = append(lines, dockerImageLines(ref, id, digests)...)
798 if preset != "" && ref != base {
799 digest := matchingDigest(base, baseDigests)
800 if digest == "" && len(baseDigests) > 0 {
801 digest = strings.TrimSpace(baseDigests[0])
802 }
803 if digest != "" {
804 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
805 } else {
806 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
807 }
808 }
809 return lines
810}
811
812func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
813 if strings.TrimSpace(image) == "" || r.Docker == "" {
814 return "", nil
815 }
816 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
817 if err != nil {
818 return "", nil
819 }
820 return parseImageInspect(out)
821}
822
823func parseImageInspect(out string) (string, []string) {
824 lines := strings.Split(out, "\n")
825 id := ""
826 if len(lines) > 0 {
827 id = strings.TrimSpace(lines[0])
828 }
829 var digests []string
830 for _, line := range lines[1:] {
831 line = strings.TrimSpace(line)
832 if line != "" {
833 digests = append(digests, line)
834 }
835 }
836 return id, digests
837}
838
839func dockerImageLines(image, id string, digests []string) []string {
840 image = strings.TrimSpace(image)
841 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
842 id = strings.TrimSpace(id)
843 digest := matchingDigest(image, digests)
844 switch {
845 case id != "" && digest != "":
846 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
847 case digest != "":
848 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
849 case id != "":
850 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
851 }
852 return lines
853}
854
855func matchingDigest(image string, digests []string) string {
856 name := imageName(image)
857 fallback := ""
858 for _, digest := range digests {
859 digest = strings.TrimSpace(digest)
860 at := strings.LastIndex(digest, "@")
861 if digest == "" || at <= 0 {
862 continue
863 }
864 if fallback == "" {
865 fallback = digest
866 }
867 repo := digest[:at]
868 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
869 return digest
870 }
871 }
872 return fallback
873}
874
875func imageName(image string) string {
876 image = strings.TrimSpace(image)
877 if i := strings.LastIndex(image, "@"); i >= 0 {
878 image = image[:i]
879 }
880 slash := strings.LastIndex(image, "/")
881 name := image
882 if slash >= 0 {
883 name = image[slash+1:]
884 }
885 if i := strings.Index(name, ":"); i >= 0 {
886 name = name[:i]
887 }
888 return name
889}
890
891func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
892 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
893 if err != nil {
894 return err
895 }
896 wanted := map[string]bool{}
897 if job.NeedsSet {
898 for _, n := range job.Needs {
899 wanted[n] = true
900 }
901 } else {
902 stageOrder := map[string]int{}
903 n := 0
904 for _, j := range jobs {
905 if _, ok := stageOrder[j.Stage]; !ok {
906 stageOrder[j.Stage] = n
907 n++
908 }
909 }
910 my := stageOrder[job.Stage]
911 for _, j := range jobs {
912 if stageOrder[j.Stage] < my {
913 wanted[j.Name] = true
914 }
915 }
916 }
917 for _, j := range jobs {
918 if !wanted[j.Name] || j.Status != "success" {
919 continue
920 }
921 for _, rel := range j.ArtifactPaths {
922 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
923 if err != nil {
924 continue
925 }
926 dst := filepath.Join(work, filepath.FromSlash(rel))
927 if err := copyFile(src, dst); err != nil {
928 return err
929 }
930 }
931 }
932 return nil
933}
934
935func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
936 var expires *time.Time
937 if job.ArtifactExpire > 0 {
938 t := time.Now().Add(job.ArtifactExpire)
939 expires = &t
940 }
941 for _, rel := range job.ArtifactPaths {
942 src := filepath.Join(work, filepath.FromSlash(rel))
943 info, err := os.Stat(src)
944 if err != nil || info.IsDir() {
945 return fmt.Errorf("artifact %s ontbreekt", rel)
946 }
947 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
948 if err := copyFile(src, dst); err != nil {
949 return err
950 }
951 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
952 return err
953 }
954 }
955 return nil
956}
957
958func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
959 name, _ := ci.SplitRef(p.Ref)
960 refProtected, err := r.refIsProtected(ctx, repo.ID, p.Ref)
961 if err != nil {
962 return nil, nil, err
963 }
964 // Fork-MR's draaien op refs/heads/forks/...: code van buiten, dus
965 // nooit groeps- of projectgeheimen en nooit als beschermde ref.
966 forkRef := ci.IsForkBranch(name)
967 if forkRef {
968 refProtected = false
969 }
970 env := map[string]string{}
971 var mask []string
972 expand := map[string]bool{}
973 if repo.OwnerKind == "group" && !forkRef {
974 if flags, err := r.Store.SecretExpandFlags(ctx, "group", repo.OwnerID); err == nil {
975 for k, on := range flags {
976 expand[k] = on
977 }
978 }
979 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
980 if err != nil {
981 return nil, nil, err
982 }
983 for k, v := range groupSecrets {
984 env[k] = v
985 mask = append(mask, v)
986 }
987 }
988 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
989 if err != nil {
990 return nil, nil, err
991 }
992 if forkRef {
993 repoSecrets = nil
994 }
995 repoFlags, _ := r.Store.SecretExpandFlags(ctx, "repo", repo.ID)
996 for k, v := range repoSecrets {
997 env[k] = v
998 mask = append(mask, v)
999 // Het project wint, ook voor de expand-vlag.
1000 expand[k] = repoFlags[k]
1001 }
1002 for k, v := range job.Variables {
1003 env[k] = v
1004 delete(expand, k)
1005 }
1006 base := strings.TrimRight(r.BaseURL, "/")
1007 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
1008 var info *ci.MRInfo
1009 if p.MRID != 0 {
1010 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
1011 target := ""
1012 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
1013 target = strings.TrimSpace(string(out))
1014 }
1015 info = &ci.MRInfo{
1016 ID: m.ID, IID: m.Number, Title: m.Title,
1017 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
1018 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
1019 TargetSHA: target,
1020 }
1021 }
1022 }
1023 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
1024 pre := ci.Variables(ci.Ref{
1025 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
1026 DefaultBranch: repo.DefaultBranch, Source: p.Source,
1027 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
1028 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
1029 })
1030 for k, v := range pre {
1031 env[k] = v
1032 }
1033 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
1034 env["CI_JOB_NAME"] = job.Name
1035 env["CI_JOB_STAGE"] = job.Stage
1036 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
1037 env["CI_JOB_TOKEN"] = token
1038 if job.Environment != "" {
1039 env["CI_ENVIRONMENT_NAME"] = job.Environment
1040 if job.EnvironmentURL != "" {
1041 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
1042 }
1043 }
1044 for k, v := range job.Variables {
1045 if strings.HasPrefix(k, "CI_NODE_") {
1046 env[k] = v
1047 }
1048 }
1049 // Variabelen met expand: na de volledige samenvoeging (groep, project,
1050 // job, CI_*). Alleen geheimen die in deze run bestaan doen mee, dus een
1051 // beschermd geheim lekt niet naar een onbeschermde ref.
1052 for k := range expand {
1053 if _, ok := env[k]; !ok || job.Variables[k] != "" {
1054 delete(expand, k)
1055 }
1056 }
1057 for k, v := range ci.ExpandVariables(env, expand) {
1058 env[k] = v
1059 if v != "" {
1060 mask = append(mask, v)
1061 }
1062 }
1063 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
1064 // en safe.directory er alsnog bij komt.
1065 markWorkspaceSafe(env)
1066 JobGitAuth(env, base+"/"+repo.FullPath(), token)
1067 mask = append(mask, token)
1068 return env, mask, nil
1069}
1070
1071// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
1072// job-token. De header geldt alleen voor de URL van dit project (met en
1073// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
1074// het in elk image, zonder bestanden in de container.
1075func JobGitAuth(env map[string]string, projectURL, token string) {
1076 if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
1077 return
1078 }
1079 n := 0
1080 if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
1081 n = v
1082 }
1083 for _, u := range []string{projectURL, projectURL + ".git"} {
1084 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
1085 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
1086 n++
1087 }
1088 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
1089}
1090
1091// markWorkspaceSafe zet safe.directory voor elke job.
1092// De checkout is van de host-uid; de container-user is dat niet.
1093// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
1094// * of /workspace zet, blijft zoals hij is.
1095func markWorkspaceSafe(env map[string]string) {
1096 const key = "safe.directory"
1097 n := 0
1098 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
1099 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
1100 n = v
1101 }
1102 }
1103 for i := 0; i < n; i++ {
1104 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
1105 continue
1106 }
1107 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
1108 case "*", "/workspace":
1109 return
1110 }
1111 }
1112 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
1113 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
1114 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
1115}
1116
1117func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
1118 netName := fmt.Sprintf("bluegit-%d", job.ID)
1119 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
1120 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
1121 }
1122 var names []string
1123 cleanup := func() {
1124 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
1125 defer cancel()
1126 for _, name := range names {
1127 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
1128 }
1129 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
1130 }
1131 for i, svc := range job.Services {
1132 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
1133 args := []string{"run", "-d", "--name", cname, "--network", netName}
1134 if svc.Alias != "" {
1135 args = append(args, "--network-alias", svc.Alias)
1136 }
1137 cmdEnv := append([]string{}, os.Environ()...)
1138 for k, v := range svc.Variables {
1139 args = append(args, "-e", k)
1140 cmdEnv = append(cmdEnv, k+"="+v)
1141 }
1142 if len(svc.Entrypoint) > 0 {
1143 args = append(args, "--entrypoint", svc.Entrypoint[0])
1144 }
1145 args = append(args, svc.Name)
1146 if len(svc.Entrypoint) > 1 {
1147 args = append(args, svc.Entrypoint[1:]...)
1148 }
1149 args = append(args, svc.Command...)
1150 cmd := exec.CommandContext(ctx, r.Docker, args...)
1151 cmd.Env = cmdEnv
1152 var stderr bytes.Buffer
1153 cmd.Stderr = &stderr
1154 if err := cmd.Run(); err != nil {
1155 cleanup()
1156 msg := strings.TrimSpace(stderr.String())
1157 if msg == "" {
1158 msg = err.Error()
1159 }
1160 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
1161 }
1162 names = append(names, cname)
1163 }
1164 return netName, cleanup, nil
1165}
1166
1167func shouldRetry(job store.Job, kind string) bool {
1168 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
1169 return false
1170 }
1171 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
1172 return true
1173 }
1174 if containsStr(job.RetryWhen, kind) {
1175 return true
1176 }
1177 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
1178}
1179
1180func classifyFailure(err error) string {
1181 msg := err.Error()
1182 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
1183 return "runner_system_failure"
1184 }
1185 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
1186 return "job_execution_timeout"
1187 }
1188 return "script_failure"
1189}
1190
1191func containsStr(list []string, s string) bool {
1192 for _, item := range list {
1193 if item == s {
1194 return true
1195 }
1196 }
1197 return false
1198}
1199
1200func safeSegment(name string) string {
1201 var b strings.Builder
1202 for _, r := range name {
1203 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1204 b.WriteRune(r)
1205 } else {
1206 b.WriteByte('_')
1207 }
1208 }
1209 if b.Len() == 0 {
1210 return "job"
1211 }
1212 return b.String()
1213}
1214
1215func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
1216 if len(job.CachePaths) == 0 {
1217 return nil
1218 }
1219 root := r.cacheDir(repoID, job.CacheKey)
1220 for _, rel := range job.CachePaths {
1221 src := filepath.Join(root, filepath.FromSlash(rel))
1222 if _, err := os.Stat(src); err != nil {
1223 continue
1224 }
1225 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
1226 return err
1227 }
1228 }
1229 return nil
1230}
1231
1232func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1233 if len(job.CachePaths) == 0 {
1234 return nil
1235 }
1236 root := r.cacheDir(repoID, job.CacheKey)
1237 for _, rel := range job.CachePaths {
1238 src := filepath.Join(work, filepath.FromSlash(rel))
1239 info, err := os.Stat(src)
1240 if err != nil || info.IsDir() {
1241 continue
1242 }
1243 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1244 return err
1245 }
1246 }
1247 return nil
1248}
1249
1250func (r *Runner) cacheDir(repoID int64, key string) string {
1251 if key == "" {
1252 key = "default"
1253 }
1254 var b strings.Builder
1255 for _, r := range key {
1256 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1257 b.WriteRune(r)
1258 }
1259 }
1260 safe := b.String()
1261 if safe == "" {
1262 safe = "default"
1263 }
1264 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1265}
1266
1267func stamp(line string) string {
1268 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1269}
1270
1271type logSink struct {
1272 mu sync.Mutex
1273 store *store.Store
1274 jobID int64
1275 out func(string)
1276 secrets []string
1277 partial string
1278 current string
1279 opened time.Time
1280}
1281
1282func (l *logSink) Write(p []byte) (int, error) {
1283 l.mu.Lock()
1284 defer l.mu.Unlock()
1285 l.partial += string(p)
1286 for {
1287 i := strings.IndexByte(l.partial, '\n')
1288 if i < 0 {
1289 break
1290 }
1291 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1292 l.partial = l.partial[i+1:]
1293 }
1294 return len(p), nil
1295}
1296
1297func (l *logSink) flush() {
1298 l.mu.Lock()
1299 defer l.mu.Unlock()
1300 if strings.TrimSpace(l.partial) != "" {
1301 l.emit(l.partial)
1302 }
1303 l.partial = ""
1304}
1305
1306func (l *logSink) emit(line string) {
1307 line = ci.Mask(line, l.secrets)
1308 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1309 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1310 dur := int(time.Since(l.opened).Seconds())
1311 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1312 l.current = ""
1313 }
1314 if strings.HasPrefix(line, "section_start:") {
1315 l.current = line
1316 l.opened = time.Now()
1317 }
1318 }
1319 if l.out != nil {
1320 l.out(stamp(line))
1321 return
1322 }
1323 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1324}
1325
1326func (l *logSink) section(name, title string) {
1327 l.emit("section_start:" + name + ":" + title)
1328}
1329
1330func (l *logSink) closeSection() {
1331 if l.current == "" {
1332 return
1333 }
1334 name := "prepare"
1335 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1336 name = parts[0]
1337 }
1338 l.emit("section_end:" + name + ":done")
1339}
1340
1341func copyFile(src, dst string) error {
1342 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1343 return err
1344 }
1345 in, err := os.Open(src)
1346 if err != nil {
1347 return err
1348 }
1349 defer in.Close()
1350 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1351 if err != nil {
1352 return err
1353 }
1354 defer out.Close()
1355 _, err = io.Copy(out, in)
1356 return err
1357}
1358
1359func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1360 if bin == "" {
1361 bin = "git"
1362 }
1363 cmd := exec.CommandContext(ctx, bin, args...)
1364 var stdout, stderr bytes.Buffer
1365 cmd.Stdout = &stdout
1366 cmd.Stderr = &stderr
1367 if err := cmd.Run(); err != nil {
1368 msg := strings.TrimSpace(stderr.String())
1369 if msg == "" {
1370 msg = err.Error()
1371 }
1372 return "", fmt.Errorf("%s", msg)
1373 }
1374 return stdout.String(), nil
1375}
1376
1377// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1378// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1379// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1380func cloneArgs(bare, work string) []string {
1381 return []string{"clone", "--no-hardlinks", bare, work}
1382}
1383
1384// ProtectedRunning is het aantal lopende doNotInterrupt-jobs in dit proces.
1385func (r *Runner) ProtectedRunning() int64 { return r.protected.Load() }
1386
1387// WaitProtected wacht bij het afsluiten tot alle lopende
1388// doNotInterrupt-jobs klaar zijn. Hun eigen timeout begrenst het wachten.
1389func (r *Runner) WaitProtected(log func(n int64)) {
1390 last := int64(-1)
1391 for n := r.protected.Load(); n > 0; n = r.protected.Load() {
1392 if n != last && log != nil {
1393 log(n)
1394 last = n
1395 }
1396 time.Sleep(time.Second)
1397 }
1398}