BlueGit

timfa bluegit

Files

bluegit / internal / runner / remote.go

remote.go

remote.go 10.2 KiB Download Raw
1package runner
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "os"
11 "path/filepath"
12 "strings"
13 "sync"
14 "time"
15
16 "gitlab.com/Timfa/bluegit/internal/ci"
17 "gitlab.com/Timfa/bluegit/internal/store"
18)
19
20// RemotePayload is wat een aparte runner bij een job krijgt.
21type RemotePayload struct {
22 JobID int64 `json:"job_id"`
23 PipelineID int64 `json:"pipeline_id"`
24 RepoID int64 `json:"repo_id"`
25 Job store.Job `json:"job"`
26 CloneURL string `json:"clone_url"`
27 SHA string `json:"sha"`
28 Script string `json:"script"`
29 Env map[string]string `json:"env"`
30 Mask []string `json:"mask"`
31 Timeout int64 `json:"timeout_seconds"`
32}
33
34var remoteMasks sync.Map // job id → []string
35
36// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.
37var RemoteOnlineGrace = 5 * time.Minute
38
39// Claim zoekt een aangeboden job voor runner rn en zet hem op running.
40// nil, nil als er niets is.
41func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {
42 if r.storage() == "" {
43 return nil, nil
44 }
45 jobs, runs, err := r.Store.OfferedJobs(ctx)
46 if err != nil {
47 return nil, err
48 }
49 cache := map[int64][]Tier{}
50 for _, j := range jobs {
51 tiers := r.tiersFor(ctx, cache, j.RepoID)
52 if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {
53 continue
54 }
55 repo, err := r.Store.RepoByID(ctx, j.RepoID)
56 if err != nil || !r.mayRun(ctx, rn, repo) {
57 continue
58 }
59 p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)
60 if err != nil || (r.Hold != nil && r.Hold(p)) {
61 continue
62 }
63 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
64 continue
65 }
66 ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)
67 if err != nil || !ok {
68 continue
69 }
70 pl, err := r.payload(ctx, repo, p, j, rn)
71 if err != nil {
72 r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())
73 continue
74 }
75 return pl, nil
76 }
77 return nil, nil
78}
79
80// mayRun is de privacycontrole: een account-runner krijgt alleen jobs
81// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit
82// repo's van die groep.
83func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool {
84 switch rn.OwnerKind {
85 case "group":
86 return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID
87 case "user":
88 if repo.OwnerKind == "user" {
89 return repo.OwnerID == rn.OwnerID
90 }
91 role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)
92 return err == nil && role.CanRead()
93 }
94 return false
95}
96
97func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {
98 token, err := r.Store.CreateJobToken(ctx, job.ID)
99 if err != nil {
100 return nil, err
101 }
102 env, mask, err := r.environment(ctx, repo, p, job, token)
103 if err != nil {
104 return nil, err
105 }
106 remoteMasks.Store(job.ID, mask)
107 limit := r.Timeout
108 if job.Timeout > 0 {
109 limit = job.Timeout
110 }
111 script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})
112 return &RemotePayload{
113 JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,
114 CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",
115 SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,
116 Timeout: int64(limit / time.Second),
117 }, nil
118}
119
120// RemoteLog voegt logregels van een aparte runner toe. cancel is true als
121// de job niet meer loopt (geannuleerd of afgebroken).
122func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {
123 if job.Status != "running" {
124 return true, nil
125 }
126 if m, ok := remoteMasks.Load(job.ID); ok {
127 chunk = ci.Mask(chunk, m.([]string))
128 }
129 if len(chunk) > 0 {
130 err = r.Store.AppendJobLog(ctx, job.ID, chunk)
131 }
132 return false, err
133}
134
135// RemoteInputs zet de artifacts van eerdere jobs in dir.
136func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {
137 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
138 if err != nil {
139 return err
140 }
141 return r.materialize(ctx, p, job, dir)
142}
143
144// RemoteArtifacts bewaart de geüploade artifacts uit dir.
145func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {
146 return r.collect(ctx, job, dir)
147}
148
149// RemoteFinish rondt een job van een aparte runner af.
150func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {
151 job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)
152 if err != nil {
153 return err
154 }
155 if job.Status != "running" {
156 return nil
157 }
158 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
159 if err != nil {
160 return err
161 }
162 var mask []string
163 if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {
164 mask = m.([]string)
165 }
166 extra, kind := "", ""
167 switch status {
168 case "success":
169 case "canceled":
170 extra = stamp("canceled")
171 default:
172 status = "failed"
173 if message == "" {
174 message = "job failed"
175 }
176 kind = classifyFailure(errors.New(message))
177 if strings.Contains(message, "job_execution_timeout") {
178 kind = "job_execution_timeout"
179 }
180 extra = stamp(ci.Mask(message, mask))
181 }
182 r.complete(p, job, p, status, extra, kind)
183 return nil
184}
185
186// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.
187func (r *Runner) reapRemote(ctx context.Context) {
188 jobs, err := r.Store.RemoteRunningJobs(ctx)
189 if err != nil {
190 return
191 }
192 now := time.Now()
193 for _, rj := range jobs {
194 rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)
195 msg := ""
196 switch {
197 case err != nil || rn.Revoked:
198 msg = "runner_system_failure: runner was removed"
199 case now.Sub(rn.LastSeen) > RemoteOnlineGrace:
200 msg = "runner_system_failure: runner " + rn.Name + " stopped responding"
201 default:
202 limit := r.Timeout
203 if rj.Job.Timeout > 0 {
204 limit = rj.Job.Timeout
205 }
206 if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {
207 msg = "job_execution_timeout: no result from runner " + rn.Name
208 }
209 }
210 if msg != "" {
211 _ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)
212 }
213 }
214}
215
216// RunRemote draait een job op de runnermachine: clone, cache, inputs,
217// container, cache opslaan. De werkmap blijft staan voor de artifacts.
218// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.
219func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
220 sink := &logSink{out: out, secrets: pl.Mask}
221 sink.section("prepare", "Preparing the docker executor")
222 _ = os.RemoveAll(work)
223 if err := os.MkdirAll(work, 0o755); err != nil {
224 sink.closeSection()
225 return err
226 }
227 if err := clone(ctx, work); err != nil {
228 sink.closeSection()
229 return fmt.Errorf("clone: %w", err)
230 }
231 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {
232 sink.closeSection()
233 return fmt.Errorf("checkout %s: %w", pl.SHA, err)
234 }
235 if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {
236 sink.closeSection()
237 return err
238 }
239 if err := inputs(ctx, work); err != nil {
240 sink.closeSection()
241 return fmt.Errorf("artifacts: %w", err)
242 }
243 if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {
244 sink.closeSection()
245 return err
246 }
247 if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {
248 return err
249 }
250 for _, rel := range pl.Job.ArtifactPaths {
251 if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {
252 return fmt.Errorf("artifact %s ontbreekt", rel)
253 }
254 }
255 return r.saveCache(pl.RepoID, pl.Job, work)
256}
257
258// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.
259// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.
260func WriteTar(w io.Writer, dir string, rels []string) error {
261 gz := gzip.NewWriter(w)
262 tw := tar.NewWriter(gz)
263 add := func(rel string) error {
264 path := filepath.Join(dir, filepath.FromSlash(rel))
265 info, err := os.Stat(path)
266 if err != nil || !info.Mode().IsRegular() {
267 return nil
268 }
269 f, err := os.Open(path)
270 if err != nil {
271 return err
272 }
273 defer f.Close()
274 if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
275 return err
276 }
277 _, err = io.Copy(tw, f)
278 return err
279 }
280 if rels == nil {
281 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
282 if err != nil || info.IsDir() {
283 return err
284 }
285 rel, _ := filepath.Rel(dir, path)
286 return add(filepath.ToSlash(rel))
287 })
288 if err != nil {
289 return err
290 }
291 } else {
292 for _, rel := range rels {
293 if err := add(rel); err != nil {
294 return err
295 }
296 }
297 }
298 if err := tw.Close(); err != nil {
299 return err
300 }
301 return gz.Close()
302}
303
304// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;
305// met allowed niet-nil alleen die paden. max begrenst de totale grootte.
306func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {
307 gz, err := gzip.NewReader(rd)
308 if err != nil {
309 return err
310 }
311 tr := tar.NewReader(gz)
312 ok := map[string]bool{}
313 for _, a := range allowed {
314 ok[filepath.ToSlash(filepath.Clean(a))] = true
315 }
316 var total int64
317 for {
318 h, err := tr.Next()
319 if errors.Is(err, io.EOF) {
320 return nil
321 }
322 if err != nil {
323 return err
324 }
325 if h.Typeflag != tar.TypeReg {
326 continue
327 }
328 name := filepath.ToSlash(filepath.Clean(h.Name))
329 if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {
330 return fmt.Errorf("unsafe path %q", h.Name)
331 }
332 if allowed != nil && !ok[name] {
333 continue
334 }
335 total += h.Size
336 if max > 0 && total > max {
337 return errors.New("archive too large")
338 }
339 dst := filepath.Join(dir, filepath.FromSlash(name))
340 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
341 return err
342 }
343 f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
344 if err != nil {
345 return err
346 }
347 if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {
348 f.Close()
349 return err
350 }
351 f.Close()
352 }
353}