BlueGit

timfa bluegit

Files

bluegit / internal / runner / remote.go

remote.go

remote.go 10.6 KiB Download Raw
1package runner
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "os"
11 "path/filepath"
12 "strings"
13 "sync"
14 "time"
15
16 "gitlab.com/Timfa/bluegit/internal/ci"
17 "gitlab.com/Timfa/bluegit/internal/store"
18)
19
20// RemotePayload is wat een aparte runner bij een job krijgt.
21type RemotePayload struct {
22 JobID int64 `json:"job_id"`
23 PipelineID int64 `json:"pipeline_id"`
24 RepoID int64 `json:"repo_id"`
25 Job store.Job `json:"job"`
26 CloneURL string `json:"clone_url"`
27 SHA string `json:"sha"`
28 Script string `json:"script"`
29 Env map[string]string `json:"env"`
30 Mask []string `json:"mask"`
31 Timeout int64 `json:"timeout_seconds"`
32}
33
34var remoteMasks sync.Map // job id → []string
35
36// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.
37var RemoteOnlineGrace = 5 * time.Minute
38
39// Claim zoekt een aangeboden job voor runner rn en zet hem op running.
40// nil, nil als er niets is.
41func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {
42 if r.storage() == "" {
43 return nil, nil
44 }
45 jobs, runs, err := r.Store.OfferedJobs(ctx)
46 if err != nil {
47 return nil, err
48 }
49 cache := map[int64][]Tier{}
50 for _, j := range jobs {
51 p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)
52 if err != nil || (r.Hold != nil && r.Hold(p)) {
53 continue
54 }
55 prot := r.refProtected(ctx, j.RepoID, p.Ref)
56 tiers := r.tiersFor(ctx, cache, j.RepoID, prot)
57 if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {
58 continue
59 }
60 repo, err := r.Store.RepoByID(ctx, j.RepoID)
61 if err != nil || !r.mayRun(ctx, rn, repo, prot) {
62 continue
63 }
64 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
65 continue
66 }
67 ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)
68 if err != nil || !ok {
69 continue
70 }
71 pl, err := r.payload(ctx, repo, p, j, rn)
72 if err != nil {
73 r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())
74 continue
75 }
76 return pl, nil
77 }
78 return nil, nil
79}
80
81// mayRun is de privacycontrole: een account-runner krijgt alleen jobs
82// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit
83// repo's van die groep.
84func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo, protected bool) bool {
85 switch rn.OwnerKind {
86 case store.RunnerGlobal:
87 // Zoals de ingebouwde runner: jobs uit alle repo's.
88 return true
89 case "group":
90 return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID
91 case "user":
92 if repo.OwnerKind == "user" {
93 return repo.OwnerID == rn.OwnerID
94 }
95 role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)
96 if err != nil || !role.CanRead() {
97 return false
98 }
99 // Beschermde ref: alleen runners van maintainers/admins.
100 return !protected || MaintainerOrAdmin(ctx, r.Store, rn.OwnerID, repo.ID)
101 }
102 return false
103}
104
105func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {
106 token, err := r.Store.CreateJobToken(ctx, job.ID)
107 if err != nil {
108 return nil, err
109 }
110 env, mask, err := r.environment(ctx, repo, p, job, token)
111 if err != nil {
112 return nil, err
113 }
114 remoteMasks.Store(job.ID, mask)
115 limit := r.Timeout
116 if job.Timeout > 0 {
117 limit = job.Timeout
118 }
119 script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})
120 return &RemotePayload{
121 JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,
122 CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",
123 SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,
124 Timeout: int64(limit / time.Second),
125 }, nil
126}
127
128// RemoteLog voegt logregels van een aparte runner toe. cancel is true als
129// de job niet meer loopt (geannuleerd of afgebroken).
130func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {
131 if job.Status != "running" {
132 return true, nil
133 }
134 if m, ok := remoteMasks.Load(job.ID); ok {
135 chunk = ci.Mask(chunk, m.([]string))
136 }
137 if len(chunk) > 0 {
138 err = r.Store.AppendJobLog(ctx, job.ID, chunk)
139 }
140 return false, err
141}
142
143// RemoteInputs zet de artifacts van eerdere jobs in dir.
144func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {
145 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
146 if err != nil {
147 return err
148 }
149 return r.materialize(ctx, p, job, dir)
150}
151
152// RemoteArtifacts bewaart de geüploade artifacts uit dir.
153func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {
154 return r.collect(ctx, job, dir)
155}
156
157// RemoteFinish rondt een job van een aparte runner af.
158func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {
159 job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)
160 if err != nil {
161 return err
162 }
163 if job.Status != "running" {
164 return nil
165 }
166 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
167 if err != nil {
168 return err
169 }
170 var mask []string
171 if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {
172 mask = m.([]string)
173 }
174 extra, kind := "", ""
175 switch status {
176 case "success":
177 case "canceled":
178 extra = stamp("canceled")
179 default:
180 status = "failed"
181 if message == "" {
182 message = "job failed"
183 }
184 kind = classifyFailure(errors.New(message))
185 if strings.Contains(message, "job_execution_timeout") {
186 kind = "job_execution_timeout"
187 }
188 extra = stamp(ci.Mask(message, mask))
189 }
190 r.complete(p, job, p, status, extra, kind)
191 return nil
192}
193
194// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.
195func (r *Runner) reapRemote(ctx context.Context) {
196 jobs, err := r.Store.RemoteRunningJobs(ctx)
197 if err != nil {
198 return
199 }
200 now := time.Now()
201 for _, rj := range jobs {
202 rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)
203 msg := ""
204 switch {
205 case err != nil || rn.Revoked:
206 msg = "runner_system_failure: runner was removed"
207 case now.Sub(rn.LastSeen) > RemoteOnlineGrace:
208 msg = "runner_system_failure: runner " + rn.Name + " stopped responding"
209 default:
210 limit := r.Timeout
211 if rj.Job.Timeout > 0 {
212 limit = rj.Job.Timeout
213 }
214 if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {
215 msg = "job_execution_timeout: no result from runner " + rn.Name
216 }
217 }
218 if msg != "" {
219 _ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)
220 }
221 }
222}
223
224// RunRemote draait een job op de runnermachine: clone, cache, inputs,
225// container, cache opslaan. De werkmap blijft staan voor de artifacts.
226// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.
227func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
228 sink := &logSink{out: out, secrets: pl.Mask}
229 sink.section("prepare", "Preparing the docker executor")
230 if err := clearWorkDir(work); err != nil {
231 sink.closeSection()
232 return fmt.Errorf("runner_system_failure: work dir: %w", err)
233 }
234 if err := os.MkdirAll(work, 0o755); err != nil {
235 sink.closeSection()
236 return err
237 }
238 if err := clone(ctx, work); err != nil {
239 sink.closeSection()
240 return fmt.Errorf("clone: %w", err)
241 }
242 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {
243 sink.closeSection()
244 return fmt.Errorf("checkout %s: %w", pl.SHA, err)
245 }
246 if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {
247 sink.closeSection()
248 return err
249 }
250 if err := inputs(ctx, work); err != nil {
251 sink.closeSection()
252 return fmt.Errorf("artifacts: %w", err)
253 }
254 if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {
255 sink.closeSection()
256 return err
257 }
258 if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {
259 return err
260 }
261 for _, rel := range pl.Job.ArtifactPaths {
262 if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {
263 return fmt.Errorf("artifact %s ontbreekt", rel)
264 }
265 }
266 return r.saveCache(pl.RepoID, pl.Job, work)
267}
268
269// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.
270// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.
271func WriteTar(w io.Writer, dir string, rels []string) error {
272 gz := gzip.NewWriter(w)
273 tw := tar.NewWriter(gz)
274 add := func(rel string) error {
275 path := filepath.Join(dir, filepath.FromSlash(rel))
276 info, err := os.Stat(path)
277 if err != nil || !info.Mode().IsRegular() {
278 return nil
279 }
280 f, err := os.Open(path)
281 if err != nil {
282 return err
283 }
284 defer f.Close()
285 if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
286 return err
287 }
288 _, err = io.Copy(tw, f)
289 return err
290 }
291 if rels == nil {
292 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
293 if err != nil || info.IsDir() {
294 return err
295 }
296 rel, _ := filepath.Rel(dir, path)
297 return add(filepath.ToSlash(rel))
298 })
299 if err != nil {
300 return err
301 }
302 } else {
303 for _, rel := range rels {
304 if err := add(rel); err != nil {
305 return err
306 }
307 }
308 }
309 if err := tw.Close(); err != nil {
310 return err
311 }
312 return gz.Close()
313}
314
315// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;
316// met allowed niet-nil alleen die paden. max begrenst de totale grootte.
317func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {
318 gz, err := gzip.NewReader(rd)
319 if err != nil {
320 return err
321 }
322 tr := tar.NewReader(gz)
323 ok := map[string]bool{}
324 for _, a := range allowed {
325 ok[filepath.ToSlash(filepath.Clean(a))] = true
326 }
327 var total int64
328 for {
329 h, err := tr.Next()
330 if errors.Is(err, io.EOF) {
331 return nil
332 }
333 if err != nil {
334 return err
335 }
336 if h.Typeflag != tar.TypeReg {
337 continue
338 }
339 name := filepath.ToSlash(filepath.Clean(h.Name))
340 if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {
341 return fmt.Errorf("unsafe path %q", h.Name)
342 }
343 if allowed != nil && !ok[name] {
344 continue
345 }
346 total += h.Size
347 if max > 0 && total > max {
348 return errors.New("archive too large")
349 }
350 dst := filepath.Join(dir, filepath.FromSlash(name))
351 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
352 return err
353 }
354 f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
355 if err != nil {
356 return err
357 }
358 if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {
359 f.Close()
360 return err
361 }
362 f.Close()
363 }
364}