BlueGit

timfa bluegit

Files

bluegit / internal / runner / remote.go

remote.go

remote.go 10.5 KiB Download Raw
1package runner
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "os"
11 "path/filepath"
12 "strings"
13 "sync"
14 "time"
15
16 "gitlab.com/Timfa/bluegit/internal/ci"
17 "gitlab.com/Timfa/bluegit/internal/store"
18)
19
20// RemotePayload is wat een aparte runner bij een job krijgt.
21type RemotePayload struct {
22 JobID int64 `json:"job_id"`
23 PipelineID int64 `json:"pipeline_id"`
24 RepoID int64 `json:"repo_id"`
25 Job store.Job `json:"job"`
26 CloneURL string `json:"clone_url"`
27 SHA string `json:"sha"`
28 Script string `json:"script"`
29 Env map[string]string `json:"env"`
30 Mask []string `json:"mask"`
31 Timeout int64 `json:"timeout_seconds"`
32}
33
34var remoteMasks sync.Map // job id → []string
35
36// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.
37var RemoteOnlineGrace = 5 * time.Minute
38
39// Claim zoekt een aangeboden job voor runner rn en zet hem op running.
40// nil, nil als er niets is.
41func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {
42 if r.storage() == "" {
43 return nil, nil
44 }
45 jobs, runs, err := r.Store.OfferedJobs(ctx)
46 if err != nil {
47 return nil, err
48 }
49 cache := map[int64][]Tier{}
50 for _, j := range jobs {
51 p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)
52 if err != nil || (r.Hold != nil && r.Hold(p)) {
53 continue
54 }
55 prot := r.refProtected(ctx, j.RepoID, p.Ref)
56 tiers := r.tiersFor(ctx, cache, j.RepoID, prot)
57 if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {
58 continue
59 }
60 repo, err := r.Store.RepoByID(ctx, j.RepoID)
61 if err != nil || !r.mayRun(ctx, rn, repo, prot) {
62 continue
63 }
64 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
65 continue
66 }
67 ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)
68 if err != nil || !ok {
69 continue
70 }
71 pl, err := r.payload(ctx, repo, p, j, rn)
72 if err != nil {
73 r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())
74 continue
75 }
76 return pl, nil
77 }
78 return nil, nil
79}
80
81// mayRun is de privacycontrole: een account-runner krijgt alleen jobs
82// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit
83// repo's van die groep.
84func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo, protected bool) bool {
85 switch rn.OwnerKind {
86 case store.RunnerGlobal:
87 // Zoals de ingebouwde runner: jobs uit alle repo's.
88 return true
89 case "group":
90 return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID
91 case "user":
92 if repo.OwnerKind == "user" {
93 return repo.OwnerID == rn.OwnerID
94 }
95 role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)
96 if err != nil || !role.CanRead() {
97 return false
98 }
99 // Beschermde ref: alleen runners van maintainers/admins.
100 return !protected || MaintainerOrAdmin(ctx, r.Store, rn.OwnerID, repo.ID)
101 }
102 return false
103}
104
105func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {
106 token, err := r.Store.CreateJobToken(ctx, job.ID)
107 if err != nil {
108 return nil, err
109 }
110 env, mask, err := r.environment(ctx, repo, p, job, token)
111 if err != nil {
112 return nil, err
113 }
114 remoteMasks.Store(job.ID, mask)
115 limit := r.Timeout
116 if job.Timeout > 0 {
117 limit = job.Timeout
118 }
119 script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})
120 return &RemotePayload{
121 JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,
122 CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",
123 SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,
124 Timeout: int64(limit / time.Second),
125 }, nil
126}
127
128// RemoteLog voegt logregels van een aparte runner toe. cancel is true als
129// de job niet meer loopt (geannuleerd of afgebroken).
130func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {
131 if job.Status != "running" {
132 return true, nil
133 }
134 if m, ok := remoteMasks.Load(job.ID); ok {
135 chunk = ci.Mask(chunk, m.([]string))
136 }
137 if len(chunk) > 0 {
138 err = r.Store.AppendJobLog(ctx, job.ID, chunk)
139 }
140 return false, err
141}
142
143// RemoteInputs zet de artifacts van eerdere jobs in dir.
144func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {
145 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
146 if err != nil {
147 return err
148 }
149 return r.materialize(ctx, p, job, dir)
150}
151
152// RemoteArtifacts bewaart de geüploade artifacts uit dir.
153func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {
154 return r.collect(ctx, job, dir)
155}
156
157// RemoteFinish rondt een job van een aparte runner af.
158func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {
159 job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)
160 if err != nil {
161 return err
162 }
163 if job.Status != "running" {
164 return nil
165 }
166 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
167 if err != nil {
168 return err
169 }
170 var mask []string
171 if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {
172 mask = m.([]string)
173 }
174 extra, kind := "", ""
175 switch status {
176 case "success":
177 case "canceled":
178 extra = stamp("canceled")
179 default:
180 status = "failed"
181 if message == "" {
182 message = "job failed"
183 }
184 kind = classifyFailure(errors.New(message))
185 if strings.Contains(message, "job_execution_timeout") {
186 kind = "job_execution_timeout"
187 }
188 extra = stamp(ci.Mask(message, mask))
189 }
190 r.complete(p, job, p, status, extra, kind)
191 return nil
192}
193
194// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.
195func (r *Runner) reapRemote(ctx context.Context) {
196 jobs, err := r.Store.RemoteRunningJobs(ctx)
197 if err != nil {
198 return
199 }
200 now := time.Now()
201 for _, rj := range jobs {
202 rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)
203 msg := ""
204 switch {
205 case err != nil || rn.Revoked:
206 msg = "runner_system_failure: runner was removed"
207 case now.Sub(rn.LastSeen) > RemoteOnlineGrace:
208 msg = "runner_system_failure: runner " + rn.Name + " stopped responding"
209 default:
210 limit := r.Timeout
211 if rj.Job.Timeout > 0 {
212 limit = rj.Job.Timeout
213 }
214 if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {
215 msg = "job_execution_timeout: no result from runner " + rn.Name
216 }
217 }
218 if msg != "" {
219 _ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)
220 }
221 }
222}
223
224// RunRemote draait een job op de runnermachine: clone, cache, inputs,
225// container, cache opslaan. De werkmap blijft staan voor de artifacts.
226// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.
227func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
228 sink := &logSink{out: out, secrets: pl.Mask}
229 sink.section("prepare", "Preparing the docker executor")
230 _ = os.RemoveAll(work)
231 if err := os.MkdirAll(work, 0o755); err != nil {
232 sink.closeSection()
233 return err
234 }
235 if err := clone(ctx, work); err != nil {
236 sink.closeSection()
237 return fmt.Errorf("clone: %w", err)
238 }
239 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {
240 sink.closeSection()
241 return fmt.Errorf("checkout %s: %w", pl.SHA, err)
242 }
243 if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {
244 sink.closeSection()
245 return err
246 }
247 if err := inputs(ctx, work); err != nil {
248 sink.closeSection()
249 return fmt.Errorf("artifacts: %w", err)
250 }
251 if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {
252 sink.closeSection()
253 return err
254 }
255 if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {
256 return err
257 }
258 for _, rel := range pl.Job.ArtifactPaths {
259 if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {
260 return fmt.Errorf("artifact %s ontbreekt", rel)
261 }
262 }
263 return r.saveCache(pl.RepoID, pl.Job, work)
264}
265
266// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.
267// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.
268func WriteTar(w io.Writer, dir string, rels []string) error {
269 gz := gzip.NewWriter(w)
270 tw := tar.NewWriter(gz)
271 add := func(rel string) error {
272 path := filepath.Join(dir, filepath.FromSlash(rel))
273 info, err := os.Stat(path)
274 if err != nil || !info.Mode().IsRegular() {
275 return nil
276 }
277 f, err := os.Open(path)
278 if err != nil {
279 return err
280 }
281 defer f.Close()
282 if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
283 return err
284 }
285 _, err = io.Copy(tw, f)
286 return err
287 }
288 if rels == nil {
289 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
290 if err != nil || info.IsDir() {
291 return err
292 }
293 rel, _ := filepath.Rel(dir, path)
294 return add(filepath.ToSlash(rel))
295 })
296 if err != nil {
297 return err
298 }
299 } else {
300 for _, rel := range rels {
301 if err := add(rel); err != nil {
302 return err
303 }
304 }
305 }
306 if err := tw.Close(); err != nil {
307 return err
308 }
309 return gz.Close()
310}
311
312// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;
313// met allowed niet-nil alleen die paden. max begrenst de totale grootte.
314func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {
315 gz, err := gzip.NewReader(rd)
316 if err != nil {
317 return err
318 }
319 tr := tar.NewReader(gz)
320 ok := map[string]bool{}
321 for _, a := range allowed {
322 ok[filepath.ToSlash(filepath.Clean(a))] = true
323 }
324 var total int64
325 for {
326 h, err := tr.Next()
327 if errors.Is(err, io.EOF) {
328 return nil
329 }
330 if err != nil {
331 return err
332 }
333 if h.Typeflag != tar.TypeReg {
334 continue
335 }
336 name := filepath.ToSlash(filepath.Clean(h.Name))
337 if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {
338 return fmt.Errorf("unsafe path %q", h.Name)
339 }
340 if allowed != nil && !ok[name] {
341 continue
342 }
343 total += h.Size
344 if max > 0 && total > max {
345 return errors.New("archive too large")
346 }
347 dst := filepath.Join(dir, filepath.FromSlash(name))
348 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
349 return err
350 }
351 f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
352 if err != nil {
353 return err
354 }
355 if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {
356 f.Close()
357 return err
358 }
359 f.Close()
360 }
361}