BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 39.0 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "os"
13 "os/exec"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "sync"
18 "sync/atomic"
19 "time"
20
21 "gitlab.com/Timfa/bluegit/internal/access"
22 "gitlab.com/Timfa/bluegit/internal/ci"
23 "gitlab.com/Timfa/bluegit/internal/gitrepo"
24 "gitlab.com/Timfa/bluegit/internal/images"
25 "gitlab.com/Timfa/bluegit/internal/runnercfg"
26 "gitlab.com/Timfa/bluegit/internal/store"
27 "gitlab.com/Timfa/bluegit/internal/webhook"
28)
29
30// Runner plant jobs en stuurt het resultaat door naar webhooks.
31type Runner struct {
32 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
33 Hold func(store.Pipeline) bool
34 Store *store.Store
35 GitBin string
36 Docker string
37 Storage string
38 BaseURL string
39 Network string
40 Name string
41 Timeout time.Duration
42 Log *slog.Logger
43 OnEvent func(repoID int64, ev webhook.Event)
44 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
45 Presets map[string]images.Override
46 // Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
47 Jobs *runnercfg.Tracker
48 ctl control
49 slots chan struct{}
50 mu sync.RWMutex
51 cancelMu sync.Mutex
52 // protected telt lopende doNotInterrupt-jobs van de ingebouwde runner.
53 protected atomic.Int64
54 cancel map[int64]context.CancelFunc
55}
56
57// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
58func (r *Runner) Interrupt(ids []int64) {
59 r.cancelMu.Lock()
60 defer r.cancelMu.Unlock()
61 for _, id := range ids {
62 if c := r.cancel[id]; c != nil {
63 c()
64 }
65 }
66}
67
68// cancelReason leest waarom een lopende job gestopt werd. CancelJob (door
69// een gebruiker) zet canceled_by_user al in de database; anders was het
70// een nieuwere pipeline op dezelfde ref.
71func (r *Runner) cancelReason(repoID, id int64) string {
72 if j, err := r.Store.JobByID(context.Background(), repoID, id); err == nil && j.Reason == "canceled_by_user" {
73 return j.Reason
74 }
75 return "canceled_by_newer_pipeline"
76}
77
78func cancelText(reason string) string {
79 if reason == "canceled_by_user" {
80 return "canceled by a user"
81 }
82 return "canceled because a newer pipeline started for the same ref"
83}
84
85func (r *Runner) track(id int64, cancel context.CancelFunc) {
86 r.cancelMu.Lock()
87 defer r.cancelMu.Unlock()
88 if r.cancel == nil {
89 r.cancel = map[int64]context.CancelFunc{}
90 }
91 r.cancel[id] = cancel
92}
93
94func (r *Runner) untrack(id int64) {
95 r.cancelMu.Lock()
96 defer r.cancelMu.Unlock()
97 delete(r.cancel, id)
98}
99
100// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
101func (r *Runner) SetStorage(path string) {
102 r.mu.Lock()
103 r.Storage = path
104 r.mu.Unlock()
105}
106
107func (r *Runner) storage() string {
108 r.mu.RLock()
109 defer r.mu.RUnlock()
110 return r.Storage
111}
112
113// Loop blijft jobs oppakken tot ctx afloopt.
114func (r *Runner) Loop(ctx context.Context, concurrency int) {
115 if concurrency < 1 {
116 concurrency = 1
117 }
118 r.slots = make(chan struct{}, maxSlots)
119 r.mu.Lock()
120 if !r.ctl.set {
121 r.ctl.limit = concurrency
122 }
123 r.mu.Unlock()
124 tick := time.NewTicker(2 * time.Second)
125 defer tick.Stop()
126 for {
127 select {
128 case <-ctx.Done():
129 return
130 case <-tick.C:
131 r.Tick(ctx)
132 }
133 }
134}
135
136// Tick bekijkt alle open pipelines één keer.
137// Zonder opslagpad gebeurt er niets.
138func (r *Runner) Tick(ctx context.Context) {
139 if r.storage() == "" {
140 return
141 }
142 pipes, err := r.Store.ActivePipelines(ctx)
143 if err != nil {
144 r.Log.Error("pipelines", "err", err)
145 return
146 }
147 for _, p := range pipes {
148 r.schedule(ctx, p)
149 }
150 r.reapRemote(ctx)
151}
152
153func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
154 r.settle(ctx, p.ID)
155 if p.Status == "running" {
156 // Een pipeline die alleen nog op handmatige jobs wacht, is niet
157 // meer bezig (ook oude pipelines van voor deze regel).
158 if st, err := r.Store.RefreshPipelineStatus(ctx, p.ID); err == nil {
159 p.Status = st
160 }
161 }
162 if r.Hold != nil && r.Hold(p) {
163 return // wacht op de self-update
164 }
165 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
166 if err != nil {
167 r.Log.Error("jobs", "err", err)
168 return
169 }
170 specs := make([]ci.Job, len(jobs))
171 status := map[string]string{}
172 for i, j := range jobs {
173 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
174 status[j.Name] = j.Status
175 }
176 tierCache := map[int64][]Tier{}
177 for _, j := range jobs {
178 if j.Status != "queued" && j.Status != "manual" {
179 continue
180 }
181 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
182 case "skip":
183 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
184 status[j.Name] = "skipped"
185 case "fail":
186 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
187 status[j.Name] = "failed"
188 case "ready":
189 if j.Status != "queued" {
190 continue
191 }
192 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
193 continue
194 }
195 // Meer jobs dan ci.concurrency blijven queued, met een plek
196 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
197 free := r.acquireSlot()
198 prot := r.refProtected(ctx, j.RepoID, p.Ref)
199 if prot {
200 r.noteProtected(ctx, j.RepoID, j)
201 }
202 if tiers := r.tiersFor(ctx, tierCache, j.RepoID, prot); len(tiers) > 1 {
203 jr := r.Store.JobRun(ctx, j.ID)
204 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
205 if !step.Builtin {
206 if free {
207 r.releaseSlot()
208 }
209 if step.Moved {
210 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
211 }
212 continue
213 }
214 }
215 if !free {
216 continue
217 }
218 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
219 if err != nil || !ok {
220 r.releaseSlot()
221 continue
222 }
223 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
224 _ = r.Store.ClearJobOffer(ctx, j.ID)
225 status[j.Name] = "running"
226 job := j
227 pipe := p
228 go r.execute(ctx, pipe, job)
229 }
230 }
231}
232
233// settle slaat jobs over die door een harde mislukking of een
234// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
235// pipelinestatus al op failed was gezet.
236func (r *Runner) settle(ctx context.Context, pipelineID int64) {
237 for {
238 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
239 if err != nil {
240 if r.Log != nil {
241 r.Log.Error("jobs", "err", err)
242 }
243 return
244 }
245 specs := make([]ci.Job, len(jobs))
246 status := map[string]string{}
247 for i, j := range jobs {
248 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
249 status[j.Name] = j.Status
250 }
251 moved := false
252 for _, j := range jobs {
253 if j.Status != "queued" && j.Status != "manual" {
254 continue
255 }
256 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
257 case "skip":
258 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
259 r.Log.Error("skip", "err", err)
260 continue
261 }
262 status[j.Name] = "skipped"
263 moved = true
264 case "fail":
265 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
266 r.Log.Error("fail", "err", err)
267 continue
268 }
269 status[j.Name] = "failed"
270 moved = true
271 }
272 }
273 if !moved {
274 return
275 }
276 }
277}
278
279// BuiltinName is de naam van de ingebouwde runner in de job-UI.
280func (r *Runner) BuiltinName() string {
281 if r.Name != "" {
282 return r.Name
283 }
284 return "BlueGit runner"
285}
286
287func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64, protected bool) []Tier {
288 key := repoID * 2
289 if protected {
290 key++
291 }
292 if t, ok := cache[key]; ok {
293 return t
294 }
295 var tiers []Tier
296 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
297 tiers, _ = Tiers(ctx, r.Store, repo, protected)
298 }
299 cache[key] = tiers
300 return tiers
301}
302
303// refProtected: draait een pipeline op een beschermde branch (en krijgt
304// hij dus beschermde geheimen)?
305func (r *Runner) refProtected(ctx context.Context, repoID int64, ref string) bool {
306 name, _ := ci.SplitRef(ref)
307 patterns, err := r.Store.ProtectedPatterns(ctx, repoID)
308 if err != nil {
309 return true // bij twijfel streng
310 }
311 return access.BranchProtected(patterns, name)
312}
313
314var protectedNoted sync.Map // job id → melding al in het log
315
316// noteProtected zet één keer een melding in het joblog als er opt-in
317// runners zijn die deze job niet mogen draaien.
318func (r *Runner) noteProtected(ctx context.Context, repoID int64, job store.Job) {
319 if _, done := protectedNoted.LoadOrStore(job.ID, true); done {
320 return
321 }
322 repo, err := r.Store.RepoByID(ctx, repoID)
323 if err != nil {
324 return
325 }
326 if names := ExcludedForProtected(ctx, r.Store, repo); len(names) > 0 {
327 _ = r.Store.AppendJobLog(ctx, job.ID, "Protected branch: this job only runs on the built-in runner, global runners, group runners and account runners of maintainers. Skipped: "+strings.Join(names, ", ")+"\n")
328 }
329}
330
331func indexOf(jobs []store.Job, name string) int {
332 for i, j := range jobs {
333 if j.Name == name {
334 return i
335 }
336 }
337 return 0
338}
339
340func (r *Runner) acquireSlot() bool {
341 if r.Paused() {
342 return false
343 }
344 if r.slots == nil {
345 return true
346 }
347 if l := r.controls().limit; l > 0 && len(r.slots) >= l {
348 return false
349 }
350 select {
351 case r.slots <- struct{}{}:
352 return true
353 default:
354 return false
355 }
356}
357
358func (r *Runner) releaseSlot() {
359 if r.slots == nil {
360 return
361 }
362 select {
363 case <-r.slots:
364 default:
365 }
366}
367
368func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
369 defer r.releaseSlot()
370 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
371 parent := ctx
372 if job.DoNotInterrupt {
373 // Een herstart of stop van BlueGit breekt deze job niet af; main
374 // wacht met afsluiten tot hij klaar is (zie WaitProtected).
375 parent = context.WithoutCancel(ctx)
376 r.protected.Add(1)
377 defer r.protected.Add(-1)
378 }
379 base, stop := context.WithCancel(parent)
380 r.track(job.ID, stop)
381 defer func() {
382 r.untrack(job.ID)
383 stop()
384 }()
385 runCtx := base
386 limit, source := EffectiveTimeout(job.Timeout, r.Timeout, r.controls().timeout)
387 r.trackStart(job)
388 if limit > 0 {
389 var cancel context.CancelFunc
390 runCtx, cancel = context.WithTimeout(base, limit)
391 defer cancel()
392 }
393 mask, err := r.runOne(runCtx, p, job)
394 status := "success"
395 extra := ""
396 kind := ""
397 switch {
398 case err == nil:
399 case parent.Err() == nil && base.Err() != nil:
400 status = "canceled"
401 kind = r.cancelReason(job.RepoID, job.ID)
402 extra = stamp(cancelText(kind))
403 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
404 status = "failed"
405 kind = "job_execution_timeout"
406 extra = stamp(TimedOutLine(limit, source))
407 default:
408 status = "failed"
409 kind = classifyFailure(err)
410 extra = stamp(ci.Mask(err.Error(), mask))
411 }
412 r.trackDone(job.ID, status)
413 r.complete(p, job, before, status, extra, kind)
414}
415
416// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
417// en webhook. Gedeeld door de ingebouwde en de aparte runner.
418func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
419 if status == "failed" && shouldRetry(job, kind) {
420 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
421 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
422 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
423 r.Log.Error("job opnieuw", "err", ferr)
424 } else if ferr == nil {
425 r.settle(context.Background(), p.ID)
426 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
427 r.Log.Error("pipelinestatus", "err", rerr)
428 }
429 return
430 }
431 }
432 if ferr := r.Store.FinishJobReason(context.Background(), job.ID, status, extra, kind); ferr != nil {
433 r.Log.Error("job afronden", "err", ferr)
434 }
435 r.settle(context.Background(), p.ID)
436 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
437 if rerr != nil {
438 r.Log.Error("pipelinestatus", "err", rerr)
439 return
440 }
441 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
442 kind := webhook.EventPipelineSuccess
443 word := "geslaagd"
444 if final == "failed" {
445 kind = webhook.EventPipelineFailure
446 word = "mislukt"
447 }
448 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
449 if err != nil {
450 return
451 }
452 branch, _ := gitrepo.BranchOfRef(p.Ref)
453 if branch == "" {
454 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
455 }
456 r.OnEvent(p.RepoID, webhook.Event{
457 Type: kind,
458 Project: repo.FullPath(),
459 Branch: branch,
460 Commit: p.CommitSHA,
461 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
462 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
463 PipelineID: fmt.Sprint(p.ID),
464 })
465 }
466}
467
468func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
469 repo, err := r.Store.RepoByID(ctx, p.RepoID)
470 if err != nil {
471 return nil, err
472 }
473 sink := &logSink{store: r.Store, jobID: job.ID}
474 sink.section("prepare", "Preparing the docker executor")
475 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
476 if err := clearWorkDir(work); err != nil {
477 sink.closeSection()
478 return nil, fmt.Errorf("runner_system_failure: work dir: %w", err)
479 }
480 if err := os.MkdirAll(work, 0o755); err != nil {
481 sink.closeSection()
482 return nil, err
483 }
484 // Na afloop (ook bij falen of annuleren) gaat de werkmap weg. Artifacts
485 // en cache staan dan al gekopieerd buiten de werkmap (collect, saveCache)
486 // en de container is gestopt. Met cleanup_work uit blijft hij staan.
487 defer r.finishWorkDir(work)
488 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
489 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
490 sink.closeSection()
491 return nil, fmt.Errorf("clone: %w", err)
492 }
493 // origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
494 // op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
495 if r.BaseURL != "" {
496 _, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
497 }
498 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
499 sink.closeSection()
500 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
501 }
502 if err := r.restoreCache(repo.ID, job, work); err != nil {
503 sink.closeSection()
504 return nil, err
505 }
506 if err := r.materialize(ctx, p, job, work); err != nil {
507 sink.closeSection()
508 return nil, err
509 }
510 scriptPath := filepath.Join(work, ".bluegit-job.sh")
511 host, _ := os.Hostname()
512 script := shellScript(job, placeNote{
513 Runner: r.Name, Host: host, WorkDir: "/workspace",
514 PipelineID: p.ID, JobID: job.ID,
515 })
516 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
517 sink.closeSection()
518 return nil, err
519 }
520 token, err := r.Store.CreateJobToken(ctx, job.ID)
521 if err != nil {
522 sink.closeSection()
523 return nil, err
524 }
525 env, mask, err := r.environment(ctx, repo, p, job, token)
526 if err != nil {
527 sink.closeSection()
528 return nil, err
529 }
530 sink.secrets = mask
531 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
532 return mask, err
533 }
534 if err := r.collect(ctx, job, work); err != nil {
535 return mask, err
536 }
537 if err := r.saveCache(repo.ID, job, work); err != nil {
538 return mask, err
539 }
540 return mask, nil
541}
542
543// execContainer haalt het image op, start services en draait het
544// jobscript in de container. De sectie prepare moet open staan.
545func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
546 ref, lines, rerr := r.resolveImage(ctx, job.Image)
547 for _, line := range lines {
548 sink.emit(line)
549 }
550 if rerr != nil {
551 sink.closeSection()
552 return rerr
553 }
554 job.Image = ref
555 job.Services = append([]store.Service(nil), job.Services...)
556 for i := range job.Services {
557 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
558 for _, line := range slines {
559 sink.emit(line)
560 }
561 if serr != nil {
562 sink.closeSection()
563 return serr
564 }
565 job.Services[i].Name = sref
566 }
567 network := r.Network
568 if network == "" {
569 network = "bridge"
570 }
571 if len(job.Services) > 0 {
572 net, cleanup, serr := r.startServices(ctx, job)
573 if serr != nil {
574 sink.closeSection()
575 return serr
576 }
577 defer cleanup()
578 network = net
579 }
580 sink.closeSection()
581 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
582 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
583 // staat de waarde niet in de proceslijst.
584 // Een vaste naam per job: een geannuleerde of afgebroken container
585 // (de docker-client stopt dan, de container niet) wordt voor de start en
586 // na afloop opgeruimd.
587 cname := fmt.Sprintf("bluegit-job-%d", job.ID)
588 removeContainer(r.Docker, cname)
589 defer removeContainer(r.Docker, cname)
590 args := []string{"run", "--rm", "--name", cname, "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
591 ctl := r.controls()
592 args = append(args, ctl.runArgs...)
593 if ctl.pull == runnercfg.PullNever {
594 args = append(args, "--pull", "never")
595 }
596 cmdEnv := append([]string{}, os.Environ()...)
597 for k, v := range env {
598 args = append(args, "-e", k)
599 cmdEnv = append(cmdEnv, k+"="+v)
600 }
601 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
602 cmd := exec.CommandContext(ctx, r.Docker, args...)
603 cmd.Env = cmdEnv
604 cmd.Stdout = sink
605 cmd.Stderr = sink
606 err := cmd.Run()
607 sink.flush()
608 if err != nil {
609 return fmt.Errorf("container ended with an error: %w", err)
610 }
611 return nil
612}
613
614func removeContainer(docker, name string) {
615 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
616 defer cancel()
617 _, _ = execOut(ctx, docker, "rm", "-f", name)
618}
619
620// finishWorkDir ruimt de werkmap van een afgelopen job op, plus de lege
621// pipelinemap erboven, tenzij de instelling cleanup_work uit staat.
622func (r *Runner) finishWorkDir(work string) {
623 if r.controls().keep {
624 return
625 }
626 _ = clearWorkDir(work)
627 _ = os.Remove(filepath.Dir(work)) // alleen als hij leeg is
628}
629
630// ClearWorkDir is clearWorkDir voor de aparte runner.
631func ClearWorkDir(work string) error { return clearWorkDir(work) }
632
633// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
634// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
635// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
636// een lege map begint. Alleen als ook dat faalt, is het een fout.
637func clearWorkDir(work string) error {
638 if err := os.RemoveAll(work); err == nil {
639 return nil
640 }
641 _ = filepath.WalkDir(work, func(p string, d fs.DirEntry, err error) error {
642 if err == nil && d.IsDir() {
643 _ = os.Chmod(p, 0o755)
644 }
645 return nil
646 })
647 if err := os.RemoveAll(work); err == nil {
648 return nil
649 }
650 stale := fmt.Sprintf("%s.stale-%d", work, time.Now().UnixNano())
651 if err := os.Rename(work, stale); err != nil {
652 return err
653 }
654 go func() { _ = os.RemoveAll(stale) }()
655 return nil
656}
657
658// placeNote zijn de vaste regels in de sectie Preparing environment.
659type placeNote struct {
660 Runner string
661 Host string
662 WorkDir string
663 PipelineID int64
664 JobID int64
665}
666
667func (n placeNote) lines() []string {
668 runner := strings.TrimSpace(n.Runner)
669 if runner == "" {
670 runner = "Steve"
671 }
672 host := strings.TrimSpace(n.Host)
673 if host == "" {
674 host = "localhost"
675 }
676 dir := strings.TrimSpace(n.WorkDir)
677 if dir == "" {
678 dir = "/workspace"
679 }
680 return []string{
681 fmt.Sprintf("Running on %s via %s...", runner, host),
682 fmt.Sprintf("Working directory: %s", dir),
683 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
684 }
685}
686
687func shSingle(s string) string {
688 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
689}
690
691func shellScript(job store.Job, note placeNote) string {
692 var b strings.Builder
693 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
694 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
695 for _, line := range note.lines() {
696 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
697 }
698 if strings.TrimSpace(job.Before) != "" {
699 b.WriteString("(\nset -e\n")
700 b.WriteString(job.Before)
701 if !strings.HasSuffix(job.Before, "\n") {
702 b.WriteByte('\n')
703 }
704 b.WriteString(")\nbrc=$?\n")
705 } else {
706 b.WriteString("brc=0\n")
707 }
708 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
709 b.WriteString("echo \"section_start:script:Running script\"\n")
710 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
711 b.WriteString(job.Script)
712 if !strings.HasSuffix(job.Script, "\n") {
713 b.WriteByte('\n')
714 }
715 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
716 b.WriteString("echo \"section_end:script:Running script\"\n")
717 b.WriteString("echo \"section_start:after:After script\"\n")
718 if strings.TrimSpace(job.After) != "" {
719 b.WriteString("(\nset -e\n")
720 b.WriteString(job.After)
721 if !strings.HasSuffix(job.After, "\n") {
722 b.WriteByte('\n')
723 }
724 b.WriteString(")\n")
725 }
726 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
727 return b.String()
728}
729
730// resolveImage kiest het image waarmee de container start.
731// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
732func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
733 image = strings.TrimSpace(image)
734 set, err := images.Effective(r.presetSet())
735 if err != nil {
736 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
737 }
738 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
739 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
740 if p, ok := images.Find(set, image); ok {
741 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
742 }
743 return image, dockerImageLines(image, "", nil), missing
744 }
745 p, ok := images.Find(set, image)
746 if !ok {
747 if r.controls().pull == runnercfg.PullAlways {
748 _, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
749 }
750 id, digests := r.inspectImage(ctx, image)
751 return image, dockerImageLines(image, id, digests), nil
752 }
753 ref, built, err := p.Ensure(ctx, r.Docker)
754 if err != nil {
755 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
756 }
757 var lines []string
758 if built {
759 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
760 }
761 id, digests := r.inspectImage(ctx, ref)
762 var baseDigests []string
763 if ref != p.From {
764 _, baseDigests = r.inspectImage(ctx, p.From)
765 }
766 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
767 return ref, lines, nil
768}
769
770func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
771 var lines []string
772 if preset != "" {
773 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
774 }
775 if ref == "" {
776 return lines
777 }
778 lines = append(lines, dockerImageLines(ref, id, digests)...)
779 if preset != "" && ref != base {
780 digest := matchingDigest(base, baseDigests)
781 if digest == "" && len(baseDigests) > 0 {
782 digest = strings.TrimSpace(baseDigests[0])
783 }
784 if digest != "" {
785 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
786 } else {
787 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
788 }
789 }
790 return lines
791}
792
793func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
794 if strings.TrimSpace(image) == "" || r.Docker == "" {
795 return "", nil
796 }
797 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
798 if err != nil {
799 return "", nil
800 }
801 return parseImageInspect(out)
802}
803
804func parseImageInspect(out string) (string, []string) {
805 lines := strings.Split(out, "\n")
806 id := ""
807 if len(lines) > 0 {
808 id = strings.TrimSpace(lines[0])
809 }
810 var digests []string
811 for _, line := range lines[1:] {
812 line = strings.TrimSpace(line)
813 if line != "" {
814 digests = append(digests, line)
815 }
816 }
817 return id, digests
818}
819
820func dockerImageLines(image, id string, digests []string) []string {
821 image = strings.TrimSpace(image)
822 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
823 id = strings.TrimSpace(id)
824 digest := matchingDigest(image, digests)
825 switch {
826 case id != "" && digest != "":
827 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
828 case digest != "":
829 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
830 case id != "":
831 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
832 }
833 return lines
834}
835
836func matchingDigest(image string, digests []string) string {
837 name := imageName(image)
838 fallback := ""
839 for _, digest := range digests {
840 digest = strings.TrimSpace(digest)
841 at := strings.LastIndex(digest, "@")
842 if digest == "" || at <= 0 {
843 continue
844 }
845 if fallback == "" {
846 fallback = digest
847 }
848 repo := digest[:at]
849 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
850 return digest
851 }
852 }
853 return fallback
854}
855
856func imageName(image string) string {
857 image = strings.TrimSpace(image)
858 if i := strings.LastIndex(image, "@"); i >= 0 {
859 image = image[:i]
860 }
861 slash := strings.LastIndex(image, "/")
862 name := image
863 if slash >= 0 {
864 name = image[slash+1:]
865 }
866 if i := strings.Index(name, ":"); i >= 0 {
867 name = name[:i]
868 }
869 return name
870}
871
872func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
873 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
874 if err != nil {
875 return err
876 }
877 wanted := map[string]bool{}
878 if job.NeedsSet {
879 for _, n := range job.Needs {
880 wanted[n] = true
881 }
882 } else {
883 stageOrder := map[string]int{}
884 n := 0
885 for _, j := range jobs {
886 if _, ok := stageOrder[j.Stage]; !ok {
887 stageOrder[j.Stage] = n
888 n++
889 }
890 }
891 my := stageOrder[job.Stage]
892 for _, j := range jobs {
893 if stageOrder[j.Stage] < my {
894 wanted[j.Name] = true
895 }
896 }
897 }
898 for _, j := range jobs {
899 if !wanted[j.Name] || j.Status != "success" {
900 continue
901 }
902 for _, rel := range j.ArtifactPaths {
903 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
904 if err != nil {
905 continue
906 }
907 dst := filepath.Join(work, filepath.FromSlash(rel))
908 if err := copyFile(src, dst); err != nil {
909 return err
910 }
911 }
912 }
913 return nil
914}
915
916func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
917 var expires *time.Time
918 if job.ArtifactExpire > 0 {
919 t := time.Now().Add(job.ArtifactExpire)
920 expires = &t
921 }
922 for _, rel := range job.ArtifactPaths {
923 src := filepath.Join(work, filepath.FromSlash(rel))
924 info, err := os.Stat(src)
925 if err != nil || info.IsDir() {
926 return fmt.Errorf("artifact %s ontbreekt", rel)
927 }
928 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
929 if err := copyFile(src, dst); err != nil {
930 return err
931 }
932 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
933 return err
934 }
935 }
936 return nil
937}
938
939func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
940 name, _ := ci.SplitRef(p.Ref)
941 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
942 if err != nil {
943 return nil, nil, err
944 }
945 refProtected := access.BranchProtected(patterns, name)
946 // Fork-MR's draaien op refs/heads/forks/...: code van buiten, dus
947 // nooit groeps- of projectgeheimen en nooit als beschermde ref.
948 forkRef := ci.IsForkBranch(name)
949 if forkRef {
950 refProtected = false
951 }
952 env := map[string]string{}
953 var mask []string
954 expand := map[string]bool{}
955 if repo.OwnerKind == "group" && !forkRef {
956 if flags, err := r.Store.SecretExpandFlags(ctx, "group", repo.OwnerID); err == nil {
957 for k, on := range flags {
958 expand[k] = on
959 }
960 }
961 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
962 if err != nil {
963 return nil, nil, err
964 }
965 for k, v := range groupSecrets {
966 env[k] = v
967 mask = append(mask, v)
968 }
969 }
970 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
971 if err != nil {
972 return nil, nil, err
973 }
974 if forkRef {
975 repoSecrets = nil
976 }
977 repoFlags, _ := r.Store.SecretExpandFlags(ctx, "repo", repo.ID)
978 for k, v := range repoSecrets {
979 env[k] = v
980 mask = append(mask, v)
981 // Het project wint, ook voor de expand-vlag.
982 expand[k] = repoFlags[k]
983 }
984 for k, v := range job.Variables {
985 env[k] = v
986 delete(expand, k)
987 }
988 base := strings.TrimRight(r.BaseURL, "/")
989 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
990 var info *ci.MRInfo
991 if p.MRID != 0 {
992 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
993 target := ""
994 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
995 target = strings.TrimSpace(string(out))
996 }
997 info = &ci.MRInfo{
998 ID: m.ID, IID: m.Number, Title: m.Title,
999 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
1000 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
1001 TargetSHA: target,
1002 }
1003 }
1004 }
1005 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
1006 pre := ci.Variables(ci.Ref{
1007 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
1008 DefaultBranch: repo.DefaultBranch, Source: p.Source,
1009 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
1010 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
1011 })
1012 for k, v := range pre {
1013 env[k] = v
1014 }
1015 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
1016 env["CI_JOB_NAME"] = job.Name
1017 env["CI_JOB_STAGE"] = job.Stage
1018 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
1019 env["CI_JOB_TOKEN"] = token
1020 if job.Environment != "" {
1021 env["CI_ENVIRONMENT_NAME"] = job.Environment
1022 if job.EnvironmentURL != "" {
1023 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
1024 }
1025 }
1026 for k, v := range job.Variables {
1027 if strings.HasPrefix(k, "CI_NODE_") {
1028 env[k] = v
1029 }
1030 }
1031 // Variabelen met expand: na de volledige samenvoeging (groep, project,
1032 // job, CI_*). Alleen geheimen die in deze run bestaan doen mee, dus een
1033 // beschermd geheim lekt niet naar een onbeschermde ref.
1034 for k := range expand {
1035 if _, ok := env[k]; !ok || job.Variables[k] != "" {
1036 delete(expand, k)
1037 }
1038 }
1039 for k, v := range ci.ExpandVariables(env, expand) {
1040 env[k] = v
1041 if v != "" {
1042 mask = append(mask, v)
1043 }
1044 }
1045 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
1046 // en safe.directory er alsnog bij komt.
1047 markWorkspaceSafe(env)
1048 JobGitAuth(env, base+"/"+repo.FullPath(), token)
1049 mask = append(mask, token)
1050 return env, mask, nil
1051}
1052
1053// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
1054// job-token. De header geldt alleen voor de URL van dit project (met en
1055// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
1056// het in elk image, zonder bestanden in de container.
1057func JobGitAuth(env map[string]string, projectURL, token string) {
1058 if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
1059 return
1060 }
1061 n := 0
1062 if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
1063 n = v
1064 }
1065 for _, u := range []string{projectURL, projectURL + ".git"} {
1066 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
1067 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
1068 n++
1069 }
1070 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
1071}
1072
1073// markWorkspaceSafe zet safe.directory voor elke job.
1074// De checkout is van de host-uid; de container-user is dat niet.
1075// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
1076// * of /workspace zet, blijft zoals hij is.
1077func markWorkspaceSafe(env map[string]string) {
1078 const key = "safe.directory"
1079 n := 0
1080 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
1081 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
1082 n = v
1083 }
1084 }
1085 for i := 0; i < n; i++ {
1086 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
1087 continue
1088 }
1089 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
1090 case "*", "/workspace":
1091 return
1092 }
1093 }
1094 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
1095 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
1096 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
1097}
1098
1099func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
1100 netName := fmt.Sprintf("bluegit-%d", job.ID)
1101 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
1102 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
1103 }
1104 var names []string
1105 cleanup := func() {
1106 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
1107 defer cancel()
1108 for _, name := range names {
1109 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
1110 }
1111 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
1112 }
1113 for i, svc := range job.Services {
1114 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
1115 args := []string{"run", "-d", "--name", cname, "--network", netName}
1116 if svc.Alias != "" {
1117 args = append(args, "--network-alias", svc.Alias)
1118 }
1119 cmdEnv := append([]string{}, os.Environ()...)
1120 for k, v := range svc.Variables {
1121 args = append(args, "-e", k)
1122 cmdEnv = append(cmdEnv, k+"="+v)
1123 }
1124 if len(svc.Entrypoint) > 0 {
1125 args = append(args, "--entrypoint", svc.Entrypoint[0])
1126 }
1127 args = append(args, svc.Name)
1128 if len(svc.Entrypoint) > 1 {
1129 args = append(args, svc.Entrypoint[1:]...)
1130 }
1131 args = append(args, svc.Command...)
1132 cmd := exec.CommandContext(ctx, r.Docker, args...)
1133 cmd.Env = cmdEnv
1134 var stderr bytes.Buffer
1135 cmd.Stderr = &stderr
1136 if err := cmd.Run(); err != nil {
1137 cleanup()
1138 msg := strings.TrimSpace(stderr.String())
1139 if msg == "" {
1140 msg = err.Error()
1141 }
1142 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
1143 }
1144 names = append(names, cname)
1145 }
1146 return netName, cleanup, nil
1147}
1148
1149func shouldRetry(job store.Job, kind string) bool {
1150 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
1151 return false
1152 }
1153 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
1154 return true
1155 }
1156 if containsStr(job.RetryWhen, kind) {
1157 return true
1158 }
1159 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
1160}
1161
1162func classifyFailure(err error) string {
1163 msg := err.Error()
1164 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
1165 return "runner_system_failure"
1166 }
1167 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
1168 return "job_execution_timeout"
1169 }
1170 return "script_failure"
1171}
1172
1173func containsStr(list []string, s string) bool {
1174 for _, item := range list {
1175 if item == s {
1176 return true
1177 }
1178 }
1179 return false
1180}
1181
1182func safeSegment(name string) string {
1183 var b strings.Builder
1184 for _, r := range name {
1185 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1186 b.WriteRune(r)
1187 } else {
1188 b.WriteByte('_')
1189 }
1190 }
1191 if b.Len() == 0 {
1192 return "job"
1193 }
1194 return b.String()
1195}
1196
1197func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
1198 if len(job.CachePaths) == 0 {
1199 return nil
1200 }
1201 root := r.cacheDir(repoID, job.CacheKey)
1202 for _, rel := range job.CachePaths {
1203 src := filepath.Join(root, filepath.FromSlash(rel))
1204 if _, err := os.Stat(src); err != nil {
1205 continue
1206 }
1207 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
1208 return err
1209 }
1210 }
1211 return nil
1212}
1213
1214func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1215 if len(job.CachePaths) == 0 {
1216 return nil
1217 }
1218 root := r.cacheDir(repoID, job.CacheKey)
1219 for _, rel := range job.CachePaths {
1220 src := filepath.Join(work, filepath.FromSlash(rel))
1221 info, err := os.Stat(src)
1222 if err != nil || info.IsDir() {
1223 continue
1224 }
1225 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1226 return err
1227 }
1228 }
1229 return nil
1230}
1231
1232func (r *Runner) cacheDir(repoID int64, key string) string {
1233 if key == "" {
1234 key = "default"
1235 }
1236 var b strings.Builder
1237 for _, r := range key {
1238 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1239 b.WriteRune(r)
1240 }
1241 }
1242 safe := b.String()
1243 if safe == "" {
1244 safe = "default"
1245 }
1246 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1247}
1248
1249func stamp(line string) string {
1250 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1251}
1252
1253type logSink struct {
1254 mu sync.Mutex
1255 store *store.Store
1256 jobID int64
1257 out func(string)
1258 secrets []string
1259 partial string
1260 current string
1261 opened time.Time
1262}
1263
1264func (l *logSink) Write(p []byte) (int, error) {
1265 l.mu.Lock()
1266 defer l.mu.Unlock()
1267 l.partial += string(p)
1268 for {
1269 i := strings.IndexByte(l.partial, '\n')
1270 if i < 0 {
1271 break
1272 }
1273 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1274 l.partial = l.partial[i+1:]
1275 }
1276 return len(p), nil
1277}
1278
1279func (l *logSink) flush() {
1280 l.mu.Lock()
1281 defer l.mu.Unlock()
1282 if strings.TrimSpace(l.partial) != "" {
1283 l.emit(l.partial)
1284 }
1285 l.partial = ""
1286}
1287
1288func (l *logSink) emit(line string) {
1289 line = ci.Mask(line, l.secrets)
1290 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1291 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1292 dur := int(time.Since(l.opened).Seconds())
1293 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1294 l.current = ""
1295 }
1296 if strings.HasPrefix(line, "section_start:") {
1297 l.current = line
1298 l.opened = time.Now()
1299 }
1300 }
1301 if l.out != nil {
1302 l.out(stamp(line))
1303 return
1304 }
1305 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1306}
1307
1308func (l *logSink) section(name, title string) {
1309 l.emit("section_start:" + name + ":" + title)
1310}
1311
1312func (l *logSink) closeSection() {
1313 if l.current == "" {
1314 return
1315 }
1316 name := "prepare"
1317 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1318 name = parts[0]
1319 }
1320 l.emit("section_end:" + name + ":done")
1321}
1322
1323func copyFile(src, dst string) error {
1324 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1325 return err
1326 }
1327 in, err := os.Open(src)
1328 if err != nil {
1329 return err
1330 }
1331 defer in.Close()
1332 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1333 if err != nil {
1334 return err
1335 }
1336 defer out.Close()
1337 _, err = io.Copy(out, in)
1338 return err
1339}
1340
1341func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1342 if bin == "" {
1343 bin = "git"
1344 }
1345 cmd := exec.CommandContext(ctx, bin, args...)
1346 var stdout, stderr bytes.Buffer
1347 cmd.Stdout = &stdout
1348 cmd.Stderr = &stderr
1349 if err := cmd.Run(); err != nil {
1350 msg := strings.TrimSpace(stderr.String())
1351 if msg == "" {
1352 msg = err.Error()
1353 }
1354 return "", fmt.Errorf("%s", msg)
1355 }
1356 return stdout.String(), nil
1357}
1358
1359// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1360// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1361// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1362func cloneArgs(bare, work string) []string {
1363 return []string{"clone", "--no-hardlinks", bare, work}
1364}
1365
1366// ProtectedRunning is het aantal lopende doNotInterrupt-jobs in dit proces.
1367func (r *Runner) ProtectedRunning() int64 { return r.protected.Load() }
1368
1369// WaitProtected wacht bij het afsluiten tot alle lopende
1370// doNotInterrupt-jobs klaar zijn. Hun eigen timeout begrenst het wachten.
1371func (r *Runner) WaitProtected(log func(n int64)) {
1372 last := int64(-1)
1373 for n := r.protected.Load(); n > 0; n = r.protected.Load() {
1374 if n != last && log != nil {
1375 log(n)
1376 last = n
1377 }
1378 time.Sleep(time.Second)
1379 }
1380}