BlueGit

timfa bluegit

Files

bluegit / internal / runner / remote.go

remote.go

remote.go 10.2 KiB Download Raw
1package runner
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "os"
11 "path/filepath"
12 "strings"
13 "sync"
14 "time"
15
16 "gitlab.com/Timfa/bluegit/internal/ci"
17 "gitlab.com/Timfa/bluegit/internal/store"
18)
19
20// RemotePayload is wat een aparte runner bij een job krijgt.
21type RemotePayload struct {
22 JobID int64 `json:"job_id"`
23 PipelineID int64 `json:"pipeline_id"`
24 RepoID int64 `json:"repo_id"`
25 Job store.Job `json:"job"`
26 CloneURL string `json:"clone_url"`
27 SHA string `json:"sha"`
28 Script string `json:"script"`
29 Env map[string]string `json:"env"`
30 Mask []string `json:"mask"`
31 Timeout int64 `json:"timeout_seconds"`
32}
33
34var remoteMasks sync.Map // job id → []string
35
36// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.
37var RemoteOnlineGrace = 5 * time.Minute
38
39// Claim zoekt een aangeboden job voor runner rn en zet hem op running.
40// nil, nil als er niets is.
41func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {
42 if r.storage() == "" {
43 return nil, nil
44 }
45 jobs, runs, err := r.Store.OfferedJobs(ctx)
46 if err != nil {
47 return nil, err
48 }
49 cache := map[int64][]Tier{}
50 for _, j := range jobs {
51 tiers := r.tiersFor(ctx, cache, j.RepoID)
52 if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {
53 continue
54 }
55 repo, err := r.Store.RepoByID(ctx, j.RepoID)
56 if err != nil || !r.mayRun(ctx, rn, repo) {
57 continue
58 }
59 p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)
60 if err != nil || (r.Hold != nil && r.Hold(p)) {
61 continue
62 }
63 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
64 continue
65 }
66 ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)
67 if err != nil || !ok {
68 continue
69 }
70 pl, err := r.payload(ctx, repo, p, j, rn)
71 if err != nil {
72 r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())
73 continue
74 }
75 return pl, nil
76 }
77 return nil, nil
78}
79
80// mayRun is de privacycontrole: een account-runner krijgt alleen jobs
81// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit
82// repo's van die groep.
83func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool {
84 switch rn.OwnerKind {
85 case store.RunnerGlobal:
86 // Zoals de ingebouwde runner: jobs uit alle repo's.
87 return true
88 case "group":
89 return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID
90 case "user":
91 if repo.OwnerKind == "user" {
92 return repo.OwnerID == rn.OwnerID
93 }
94 role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)
95 return err == nil && role.CanRead()
96 }
97 return false
98}
99
100func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {
101 token, err := r.Store.CreateJobToken(ctx, job.ID)
102 if err != nil {
103 return nil, err
104 }
105 env, mask, err := r.environment(ctx, repo, p, job, token)
106 if err != nil {
107 return nil, err
108 }
109 remoteMasks.Store(job.ID, mask)
110 limit := r.Timeout
111 if job.Timeout > 0 {
112 limit = job.Timeout
113 }
114 script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})
115 return &RemotePayload{
116 JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,
117 CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",
118 SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,
119 Timeout: int64(limit / time.Second),
120 }, nil
121}
122
123// RemoteLog voegt logregels van een aparte runner toe. cancel is true als
124// de job niet meer loopt (geannuleerd of afgebroken).
125func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {
126 if job.Status != "running" {
127 return true, nil
128 }
129 if m, ok := remoteMasks.Load(job.ID); ok {
130 chunk = ci.Mask(chunk, m.([]string))
131 }
132 if len(chunk) > 0 {
133 err = r.Store.AppendJobLog(ctx, job.ID, chunk)
134 }
135 return false, err
136}
137
138// RemoteInputs zet de artifacts van eerdere jobs in dir.
139func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {
140 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
141 if err != nil {
142 return err
143 }
144 return r.materialize(ctx, p, job, dir)
145}
146
147// RemoteArtifacts bewaart de geüploade artifacts uit dir.
148func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {
149 return r.collect(ctx, job, dir)
150}
151
152// RemoteFinish rondt een job van een aparte runner af.
153func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {
154 job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)
155 if err != nil {
156 return err
157 }
158 if job.Status != "running" {
159 return nil
160 }
161 p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
162 if err != nil {
163 return err
164 }
165 var mask []string
166 if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {
167 mask = m.([]string)
168 }
169 extra, kind := "", ""
170 switch status {
171 case "success":
172 case "canceled":
173 extra = stamp("canceled")
174 default:
175 status = "failed"
176 if message == "" {
177 message = "job failed"
178 }
179 kind = classifyFailure(errors.New(message))
180 if strings.Contains(message, "job_execution_timeout") {
181 kind = "job_execution_timeout"
182 }
183 extra = stamp(ci.Mask(message, mask))
184 }
185 r.complete(p, job, p, status, extra, kind)
186 return nil
187}
188
189// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.
190func (r *Runner) reapRemote(ctx context.Context) {
191 jobs, err := r.Store.RemoteRunningJobs(ctx)
192 if err != nil {
193 return
194 }
195 now := time.Now()
196 for _, rj := range jobs {
197 rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)
198 msg := ""
199 switch {
200 case err != nil || rn.Revoked:
201 msg = "runner_system_failure: runner was removed"
202 case now.Sub(rn.LastSeen) > RemoteOnlineGrace:
203 msg = "runner_system_failure: runner " + rn.Name + " stopped responding"
204 default:
205 limit := r.Timeout
206 if rj.Job.Timeout > 0 {
207 limit = rj.Job.Timeout
208 }
209 if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {
210 msg = "job_execution_timeout: no result from runner " + rn.Name
211 }
212 }
213 if msg != "" {
214 _ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)
215 }
216 }
217}
218
219// RunRemote draait een job op de runnermachine: clone, cache, inputs,
220// container, cache opslaan. De werkmap blijft staan voor de artifacts.
221// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.
222func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
223 sink := &logSink{out: out, secrets: pl.Mask}
224 sink.section("prepare", "Preparing the docker executor")
225 _ = os.RemoveAll(work)
226 if err := os.MkdirAll(work, 0o755); err != nil {
227 sink.closeSection()
228 return err
229 }
230 if err := clone(ctx, work); err != nil {
231 sink.closeSection()
232 return fmt.Errorf("clone: %w", err)
233 }
234 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {
235 sink.closeSection()
236 return fmt.Errorf("checkout %s: %w", pl.SHA, err)
237 }
238 if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {
239 sink.closeSection()
240 return err
241 }
242 if err := inputs(ctx, work); err != nil {
243 sink.closeSection()
244 return fmt.Errorf("artifacts: %w", err)
245 }
246 if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {
247 sink.closeSection()
248 return err
249 }
250 if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {
251 return err
252 }
253 for _, rel := range pl.Job.ArtifactPaths {
254 if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {
255 return fmt.Errorf("artifact %s ontbreekt", rel)
256 }
257 }
258 return r.saveCache(pl.RepoID, pl.Job, work)
259}
260
261// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.
262// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.
263func WriteTar(w io.Writer, dir string, rels []string) error {
264 gz := gzip.NewWriter(w)
265 tw := tar.NewWriter(gz)
266 add := func(rel string) error {
267 path := filepath.Join(dir, filepath.FromSlash(rel))
268 info, err := os.Stat(path)
269 if err != nil || !info.Mode().IsRegular() {
270 return nil
271 }
272 f, err := os.Open(path)
273 if err != nil {
274 return err
275 }
276 defer f.Close()
277 if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
278 return err
279 }
280 _, err = io.Copy(tw, f)
281 return err
282 }
283 if rels == nil {
284 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
285 if err != nil || info.IsDir() {
286 return err
287 }
288 rel, _ := filepath.Rel(dir, path)
289 return add(filepath.ToSlash(rel))
290 })
291 if err != nil {
292 return err
293 }
294 } else {
295 for _, rel := range rels {
296 if err := add(rel); err != nil {
297 return err
298 }
299 }
300 }
301 if err := tw.Close(); err != nil {
302 return err
303 }
304 return gz.Close()
305}
306
307// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;
308// met allowed niet-nil alleen die paden. max begrenst de totale grootte.
309func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {
310 gz, err := gzip.NewReader(rd)
311 if err != nil {
312 return err
313 }
314 tr := tar.NewReader(gz)
315 ok := map[string]bool{}
316 for _, a := range allowed {
317 ok[filepath.ToSlash(filepath.Clean(a))] = true
318 }
319 var total int64
320 for {
321 h, err := tr.Next()
322 if errors.Is(err, io.EOF) {
323 return nil
324 }
325 if err != nil {
326 return err
327 }
328 if h.Typeflag != tar.TypeReg {
329 continue
330 }
331 name := filepath.ToSlash(filepath.Clean(h.Name))
332 if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {
333 return fmt.Errorf("unsafe path %q", h.Name)
334 }
335 if allowed != nil && !ok[name] {
336 continue
337 }
338 total += h.Size
339 if max > 0 && total > max {
340 return errors.New("archive too large")
341 }
342 dst := filepath.Join(dir, filepath.FromSlash(name))
343 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
344 return err
345 }
346 f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
347 if err != nil {
348 return err
349 }
350 if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {
351 f.Close()
352 return err
353 }
354 f.Close()
355 }
356}