BlueGit

timfa bluegit

Merged

Runner: werkmap na afloop opruimen

!72 · merged 3 weeks ago

timfa wants to merge 1 commit from ada/runner-workdir-cleanup into stage

Pipeline

#278 Passed
Stage: build

Reviews

  • No reviews yet.

Merged by timfa on 25 October 2026 6406a3c1

Overview Commits 1 Pipelines 1 Changes 5

Changes

5 files changed +62 −3
cmd/bluegit-runner/main.go +1 −1
@@ -515,7 +515,7 @@ func (c *client) runJob(parent context.Context, pl runner.RemotePayload) {
515 515 work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))
516 516 defer func() {
517 517 if st.CleanupWork {
518 _ = os.RemoveAll(work)
518 _ = runner.ClearWorkDir(work)
519 519 }
520 520 runnercfg.Cleanup(context.Background(), c.cfg.Docker, c.settings(), c.jobs, filepath.Join(c.cfg.WorkDir, "cache"))
521 521 }()
@@ -515,7 +515,7 @@ func (c *client) runJob(parent context.Context, pl runner.RemotePayload) {
515 work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))
515 work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))
516 defer func() {
516 defer func() {
517 if st.CleanupWork {
517 if st.CleanupWork {
518 _ = os.RemoveAll(work)
518 _ = runner.ClearWorkDir(work)
519 }
519 }
520 runnercfg.Cleanup(context.Background(), c.cfg.Docker, c.settings(), c.jobs, filepath.Join(c.cfg.WorkDir, "cache"))
520 runnercfg.Cleanup(context.Background(), c.cfg.Docker, c.settings(), c.jobs, filepath.Join(c.cfg.WorkDir, "cache"))
521 }()
521 }()
internal/runner/control.go +2 −1
@@ -23,6 +23,7 @@ type control struct {
23 23 pull string
24 24 timeout int
25 25 presets map[string]images.Override
26 keep bool // werkmappen na afloop bewaren (cleanup_work uit)
26 27 set bool
27 28 }
28 29
@@ -31,7 +32,7 @@ type control struct {
31 32 func (r *Runner) Configure(s runnercfg.Settings, presets map[string]images.Override) {
32 33 r.mu.Lock()
33 34 defer r.mu.Unlock()
34 r.ctl = control{limit: s.Concurrency, runArgs: s.RunArgs(), pull: s.PullPolicy, timeout: s.JobTimeoutMin, presets: presets, set: true}
35 r.ctl = control{limit: s.Concurrency, runArgs: s.RunArgs(), pull: s.PullPolicy, timeout: s.JobTimeoutMin, presets: presets, keep: !s.CleanupWork, set: true}
35 36 if n := strings.TrimSpace(s.Name); n != "" {
36 37 r.Name = n
37 38 }
@@ -23,6 +23,7 @@ type control struct {
23 pull string
23 pull string
24 timeout int
24 timeout int
25 presets map[string]images.Override
25 presets map[string]images.Override
26 keep bool // werkmappen na afloop bewaren (cleanup_work uit)
26 set bool
27 set bool
27}
28}
28
29
@@ -31,7 +32,7 @@ type control struct {
31func (r *Runner) Configure(s runnercfg.Settings, presets map[string]images.Override) {
32func (r *Runner) Configure(s runnercfg.Settings, presets map[string]images.Override) {
32 r.mu.Lock()
33 r.mu.Lock()
33 defer r.mu.Unlock()
34 defer r.mu.Unlock()
34 r.ctl = control{limit: s.Concurrency, runArgs: s.RunArgs(), pull: s.PullPolicy, timeout: s.JobTimeoutMin, presets: presets, set: true}
35 r.ctl = control{limit: s.Concurrency, runArgs: s.RunArgs(), pull: s.PullPolicy, timeout: s.JobTimeoutMin, presets: presets, keep: !s.CleanupWork, set: true}
35 if n := strings.TrimSpace(s.Name); n != "" {
36 if n := strings.TrimSpace(s.Name); n != "" {
36 r.Name = n
37 r.Name = n
37 }
38 }
internal/runner/remote.go +4 −1
@@ -227,7 +227,10 @@ func (r *Runner) reapRemote(ctx context.Context) {
227 227 func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
228 228 sink := &logSink{out: out, secrets: pl.Mask}
229 229 sink.section("prepare", "Preparing the docker executor")
230 _ = os.RemoveAll(work)
230 if err := clearWorkDir(work); err != nil {
231 sink.closeSection()
232 return fmt.Errorf("runner_system_failure: work dir: %w", err)
233 }
231 234 if err := os.MkdirAll(work, 0o755); err != nil {
232 235 sink.closeSection()
233 236 return err
@@ -227,7 +227,10 @@ func (r *Runner) reapRemote(ctx context.Context) {
227func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
227func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
228 sink := &logSink{out: out, secrets: pl.Mask}
228 sink := &logSink{out: out, secrets: pl.Mask}
229 sink.section("prepare", "Preparing the docker executor")
229 sink.section("prepare", "Preparing the docker executor")
230 _ = os.RemoveAll(work)
230 if err := clearWorkDir(work); err != nil {
231 sink.closeSection()
232 return fmt.Errorf("runner_system_failure: work dir: %w", err)
233 }
231 if err := os.MkdirAll(work, 0o755); err != nil {
234 if err := os.MkdirAll(work, 0o755); err != nil {
232 sink.closeSection()
235 sink.closeSection()
233 return err
236 return err
internal/runner/runner.go +17 −0
@@ -451,6 +451,10 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
451 451 sink.closeSection()
452 452 return nil, err
453 453 }
454 // Na afloop (ook bij falen of annuleren) gaat de werkmap weg. Artifacts
455 // en cache staan dan al gekopieerd buiten de werkmap (collect, saveCache)
456 // en de container is gestopt. Met cleanup_work uit blijft hij staan.
457 defer r.finishWorkDir(work)
454 458 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
455 459 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
456 460 sink.closeSection()
@@ -583,6 +587,19 @@ func removeContainer(docker, name string) {
583 587 _, _ = execOut(ctx, docker, "rm", "-f", name)
584 588 }
585 589
590 // finishWorkDir ruimt de werkmap van een afgelopen job op, plus de lege
591 // pipelinemap erboven, tenzij de instelling cleanup_work uit staat.
592 func (r *Runner) finishWorkDir(work string) {
593 if r.controls().keep {
594 return
595 }
596 _ = clearWorkDir(work)
597 _ = os.Remove(filepath.Dir(work)) // alleen als hij leeg is
598 }
599
600 // ClearWorkDir is clearWorkDir voor de aparte runner.
601 func ClearWorkDir(work string) error { return clearWorkDir(work) }
602
586 603 // clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
587 604 // container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
588 605 // niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
@@ -451,6 +451,10 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
451 sink.closeSection()
451 sink.closeSection()
452 return nil, err
452 return nil, err
453 }
453 }
454 // Na afloop (ook bij falen of annuleren) gaat de werkmap weg. Artifacts
455 // en cache staan dan al gekopieerd buiten de werkmap (collect, saveCache)
456 // en de container is gestopt. Met cleanup_work uit blijft hij staan.
457 defer r.finishWorkDir(work)
454 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
458 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
455 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
459 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
456 sink.closeSection()
460 sink.closeSection()
@@ -583,6 +587,19 @@ func removeContainer(docker, name string) {
583 _, _ = execOut(ctx, docker, "rm", "-f", name)
587 _, _ = execOut(ctx, docker, "rm", "-f", name)
584}
588}
585
589
590// finishWorkDir ruimt de werkmap van een afgelopen job op, plus de lege
591// pipelinemap erboven, tenzij de instelling cleanup_work uit staat.
592func (r *Runner) finishWorkDir(work string) {
593 if r.controls().keep {
594 return
595 }
596 _ = clearWorkDir(work)
597 _ = os.Remove(filepath.Dir(work)) // alleen als hij leeg is
598}
599
600// ClearWorkDir is clearWorkDir voor de aparte runner.
601func ClearWorkDir(work string) error { return clearWorkDir(work) }
602
586// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
603// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
587// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
604// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
588// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
605// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
internal/runner/workdir_test.go +38 −0
@@ -36,3 +36,41 @@ func TestClearWorkDirMissing(t *testing.T) {
36 36 t.Fatal(err)
37 37 }
38 38 }
39
40 // Na afloop verdwijnt de werkmap (en de lege pipelinemap), behalve met
41 // cleanup_work uit.
42 func TestFinishWorkDir(t *testing.T) {
43 for _, keep := range []bool{false, true} {
44 root := t.TempDir()
45 work := filepath.Join(root, "work", "7", "build")
46 if err := os.MkdirAll(filepath.Join(work, "sub"), 0o755); err != nil {
47 t.Fatal(err)
48 }
49 r := &Runner{}
50 r.ctl = control{keep: keep, set: true}
51 r.finishWorkDir(work)
52 _, err := os.Stat(filepath.Dir(work))
53 if keep && err != nil {
54 t.Fatalf("keep: work removed: %v", err)
55 }
56 if !keep && !os.IsNotExist(err) {
57 t.Fatalf("pipeline dir still there: %v", err)
58 }
59 }
60 }
61
62 // De pipelinemap blijft staan zolang een andere job er nog in werkt.
63 func TestFinishWorkDirSibling(t *testing.T) {
64 root := t.TempDir()
65 a := filepath.Join(root, "work", "7", "a")
66 b := filepath.Join(root, "work", "7", "b")
67 _ = os.MkdirAll(a, 0o755)
68 _ = os.MkdirAll(b, 0o755)
69 (&Runner{}).finishWorkDir(a)
70 if _, err := os.Stat(a); !os.IsNotExist(err) {
71 t.Fatal("a still there")
72 }
73 if _, err := os.Stat(b); err != nil {
74 t.Fatal("b removed")
75 }
76 }
@@ -36,3 +36,41 @@ func TestClearWorkDirMissing(t *testing.T) {
36 t.Fatal(err)
36 t.Fatal(err)
37 }
37 }
38}
38}
39
40// Na afloop verdwijnt de werkmap (en de lege pipelinemap), behalve met
41// cleanup_work uit.
42func TestFinishWorkDir(t *testing.T) {
43 for _, keep := range []bool{false, true} {
44 root := t.TempDir()
45 work := filepath.Join(root, "work", "7", "build")
46 if err := os.MkdirAll(filepath.Join(work, "sub"), 0o755); err != nil {
47 t.Fatal(err)
48 }
49 r := &Runner{}
50 r.ctl = control{keep: keep, set: true}
51 r.finishWorkDir(work)
52 _, err := os.Stat(filepath.Dir(work))
53 if keep && err != nil {
54 t.Fatalf("keep: work removed: %v", err)
55 }
56 if !keep && !os.IsNotExist(err) {
57 t.Fatalf("pipeline dir still there: %v", err)
58 }
59 }
60}
61
62// De pipelinemap blijft staan zolang een andere job er nog in werkt.
63func TestFinishWorkDirSibling(t *testing.T) {
64 root := t.TempDir()
65 a := filepath.Join(root, "work", "7", "a")
66 b := filepath.Join(root, "work", "7", "b")
67 _ = os.MkdirAll(a, 0o755)
68 _ = os.MkdirAll(b, 0o755)
69 (&Runner{}).finishWorkDir(a)
70 if _, err := os.Stat(a); !os.IsNotExist(err) {
71 t.Fatal("a still there")
72 }
73 if _, err := os.Stat(b); err != nil {
74 t.Fatal("b removed")
75 }
76}