BlueGit

timfa bluegit

Files

bluegit / internal / runner / remote.go

remote.go

remote.go 10.8 KiB Download Raw
1package runner
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "os"
11 "path/filepath"
12 "strings"
13 "sync"
14 "time"
15
16 "gitlab.com/Timfa/bluegit/internal/ci"
17 "gitlab.com/Timfa/bluegit/internal/store"
18)
19
20// RemotePayload is wat een aparte runner bij een job krijgt.
21type RemotePayload struct {
22 JobID int64 `json:"job_id"`
23 PipelineID int64 `json:"pipeline_id"`
24 RepoID int64 `json:"repo_id"`
25 Job store.Job `json:"job"`
26 CloneURL string `json:"clone_url"`
27 SHA string `json:"sha"`
28 Script string `json:"script"`
29 Env map[string]string `json:"env"`
30 Mask []string `json:"mask"`
31 Timeout int64 `json:"timeout_seconds"`
32}
33
34var remoteMasks sync.Map // job id → []string
35
36// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.
37var RemoteOnlineGrace = 5 * time.Minute
38
39// Claim zoekt een aangeboden job voor runner rn en zet hem op running.
40// nil, nil als er niets is.
41func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {
42 if r.storage() == "" {
43 return nil, nil
44 }
45 jobs, runs, err := r.Store.OfferedJobs(ctx)
46 if err != nil {
47 return nil, err
48 }
49 cache := map[int64][]Tier{}
50 for _, j := range jobs {
51 p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)
52 if err != nil || (r.Hold != nil && r.Hold(p)) {
53 continue
54 }
55 prot := r.refProtected(ctx, j.RepoID, p.Ref)
56 tiers := r.tiersFor(ctx, cache, j.RepoID, prot)
57 if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {
58 continue
59 }
60 repo, err := r.Store.RepoByID(ctx, j.RepoID)
61 if err != nil || !r.mayRun(ctx, rn, repo, prot) {
62 continue
63 }
64 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
65 continue
66 }
67 ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)
68 if err != nil || !ok {
69 continue
70 }
71 pl, err := r.payload(ctx, repo, p, j, rn)
72 if err != nil {
73 r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())
74 continue
75 }
76 return pl, nil
77 }
78 return nil, nil
79}
80
81// mayRun is de privacycontrole: een account-runner krijgt alleen jobs
82// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit
83// repo's van die groep.
84func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo, protected bool) bool {
85 switch rn.OwnerKind {
86 case store.RunnerGlobal:
87 // Zoals de ingebouwde runner: jobs uit alle repo's.
88 return true
89 case "group":
90 return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID
91 case "user":
92 if repo.OwnerKind == "user" {
93 return repo.OwnerID == rn.OwnerID
94 }
95 role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)
96 if err != nil || !role.CanRead() {
97 return false
98 }
99 // Beschermde ref: alleen runners van maintainers/admins.
100 return !protected || MaintainerOrAdmin(ctx, r.Store, rn.OwnerID, repo.ID)
101 }
102 return false
103}
104
105func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {
106 token, err := r.Store.CreateJobToken(ctx, job.ID)
107 if err != nil {
108 return nil, err
109 }
110 env, mask, err := r.environment(ctx, repo, p, job, token)
111 if err != nil {
112 return nil, err
113 }
114 remoteMasks.Store(job.ID, mask)
115 limit := r.Timeout
116 if job.Timeout > 0 {
117 limit = job.Timeout
118 }
119 script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})
120 return &RemotePayload{
121 JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,
122 CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",
123 SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,
124 Timeout: int64(limit / time.Second),
125 }, nil
126}
127
128// RemoteLog voegt logregels van een aparte runner toe. cancel is true als
129// de job niet meer loopt (geannuleerd of afgebroken).
130func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {
131 if job.Status != "running" {
132 return true, nil
133 }
134 if m, ok := remoteMasks.Load(job.ID); ok {
135 chunk = ci.Mask(chunk, m.([]string))
136 }
137 if len(chunk) > 0 {
138 err = r.Store.AppendJobLog(ctx, job.ID, chunk)
139 }
140 return false, err
141}
142
143// RemoteInputs zet de artifacts van eerdere jobs in dir.
144func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {
145 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
146 if err != nil {
147 return err
148 }
149 return r.materialize(ctx, p, job, dir)
150}
151
152// RemoteArtifacts bewaart de geüploade artifacts uit dir.
153func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {
154 return r.collect(ctx, job, dir)
155}
156
157// RemoteFinish rondt een job van een aparte runner af.
158func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {
159 job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)
160 if err != nil {
161 return err
162 }
163 if job.Status != "running" {
164 return nil
165 }
166 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
167 if err != nil {
168 return err
169 }
170 var mask []string
171 if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {
172 mask = m.([]string)
173 }
174 extra, kind := "", ""
175 switch status {
176 case "success":
177 case "canceled":
178 extra = stamp("canceled")
179 default:
180 status = "failed"
181 if message == "" {
182 message = "job failed"
183 }
184 kind = classifyFailure(errors.New(message))
185 if strings.Contains(message, "job_execution_timeout") {
186 kind = "job_execution_timeout"
187 }
188 if kind == "job_execution_timeout" {
189 message = strings.TrimPrefix(message, "job_execution_timeout: ")
190 }
191 extra = stamp(ci.Mask(message, mask))
192 }
193 r.complete(p, job, p, status, extra, kind)
194 return nil
195}
196
197// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.
198func (r *Runner) reapRemote(ctx context.Context) {
199 jobs, err := r.Store.RemoteRunningJobs(ctx)
200 if err != nil {
201 return
202 }
203 now := time.Now()
204 for _, rj := range jobs {
205 rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)
206 msg := ""
207 switch {
208 case err != nil || rn.Revoked:
209 msg = "runner_system_failure: runner was removed"
210 case now.Sub(rn.LastSeen) > RemoteOnlineGrace && !rj.Job.DoNotInterrupt:
211 // doNotInterrupt: de job kan op de runner gewoon doorlopen als
212 // alleen de verbinding weg is; dan geldt alleen de timeout hieronder.
213 msg = "runner_system_failure: runner " + rn.Name + " stopped responding"
214 default:
215 limit := r.Timeout
216 if rj.Job.Timeout > 0 {
217 limit = rj.Job.Timeout
218 }
219 if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {
220 msg = "job_execution_timeout: no result from runner " + rn.Name
221 }
222 }
223 if msg != "" {
224 _ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)
225 }
226 }
227}
228
229// RunRemote draait een job op de runnermachine: clone, cache, inputs,
230// container, cache opslaan. De werkmap blijft staan voor de artifacts.
231// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.
232func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
233 sink := &logSink{out: out, secrets: pl.Mask}
234 sink.section("prepare", "Preparing the docker executor")
235 if err := clearWorkDir(work); err != nil {
236 sink.closeSection()
237 return fmt.Errorf("runner_system_failure: work dir: %w", err)
238 }
239 if err := os.MkdirAll(work, 0o755); err != nil {
240 sink.closeSection()
241 return err
242 }
243 if err := clone(ctx, work); err != nil {
244 sink.closeSection()
245 return fmt.Errorf("clone: %w", err)
246 }
247 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {
248 sink.closeSection()
249 return fmt.Errorf("checkout %s: %w", pl.SHA, err)
250 }
251 if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {
252 sink.closeSection()
253 return err
254 }
255 if err := inputs(ctx, work); err != nil {
256 sink.closeSection()
257 return fmt.Errorf("artifacts: %w", err)
258 }
259 if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {
260 sink.closeSection()
261 return err
262 }
263 if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {
264 return err
265 }
266 for _, rel := range pl.Job.ArtifactPaths {
267 if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {
268 return fmt.Errorf("artifact %s ontbreekt", rel)
269 }
270 }
271 return r.saveCache(pl.RepoID, pl.Job, work)
272}
273
274// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.
275// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.
276func WriteTar(w io.Writer, dir string, rels []string) error {
277 gz := gzip.NewWriter(w)
278 tw := tar.NewWriter(gz)
279 add := func(rel string) error {
280 path := filepath.Join(dir, filepath.FromSlash(rel))
281 info, err := os.Stat(path)
282 if err != nil || !info.Mode().IsRegular() {
283 return nil
284 }
285 f, err := os.Open(path)
286 if err != nil {
287 return err
288 }
289 defer f.Close()
290 if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
291 return err
292 }
293 _, err = io.Copy(tw, f)
294 return err
295 }
296 if rels == nil {
297 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
298 if err != nil || info.IsDir() {
299 return err
300 }
301 rel, _ := filepath.Rel(dir, path)
302 return add(filepath.ToSlash(rel))
303 })
304 if err != nil {
305 return err
306 }
307 } else {
308 for _, rel := range rels {
309 if err := add(rel); err != nil {
310 return err
311 }
312 }
313 }
314 if err := tw.Close(); err != nil {
315 return err
316 }
317 return gz.Close()
318}
319
320// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;
321// met allowed niet-nil alleen die paden. max begrenst de totale grootte.
322func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {
323 gz, err := gzip.NewReader(rd)
324 if err != nil {
325 return err
326 }
327 tr := tar.NewReader(gz)
328 ok := map[string]bool{}
329 for _, a := range allowed {
330 ok[filepath.ToSlash(filepath.Clean(a))] = true
331 }
332 var total int64
333 for {
334 h, err := tr.Next()
335 if errors.Is(err, io.EOF) {
336 return nil
337 }
338 if err != nil {
339 return err
340 }
341 if h.Typeflag != tar.TypeReg {
342 continue
343 }
344 name := filepath.ToSlash(filepath.Clean(h.Name))
345 if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {
346 return fmt.Errorf("unsafe path %q", h.Name)
347 }
348 if allowed != nil && !ok[name] {
349 continue
350 }
351 total += h.Size
352 if max > 0 && total > max {
353 return errors.New("archive too large")
354 }
355 dst := filepath.Join(dir, filepath.FromSlash(name))
356 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
357 return err
358 }
359 f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
360 if err != nil {
361 return err
362 }
363 if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {
364 f.Close()
365 return err
366 }
367 f.Close()
368 }
369}