| 1 | package runner |
| 2 | |
| 3 | import ( |
| 4 | "archive/tar" |
| 5 | "compress/gzip" |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "io" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "sync" |
| 14 | "time" |
| 15 | |
| 16 | "gitlab.com/Timfa/bluegit/internal/ci" |
| 17 | "gitlab.com/Timfa/bluegit/internal/store" |
| 18 | ) |
| 19 | |
| 20 | // RemotePayload is wat een aparte runner bij een job krijgt. |
| 21 | type RemotePayload struct { |
| 22 | JobID int64 `json:"job_id"` |
| 23 | PipelineID int64 `json:"pipeline_id"` |
| 24 | RepoID int64 `json:"repo_id"` |
| 25 | Job store.Job `json:"job"` |
| 26 | CloneURL string `json:"clone_url"` |
| 27 | SHA string `json:"sha"` |
| 28 | Script string `json:"script"` |
| 29 | Env map[string]string `json:"env"` |
| 30 | Mask []string `json:"mask"` |
| 31 | Timeout int64 `json:"timeout_seconds"` |
| 32 | } |
| 33 | |
| 34 | var remoteMasks sync.Map // job id → []string |
| 35 | |
| 36 | // RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job. |
| 37 | var RemoteOnlineGrace = 5 * time.Minute |
| 38 | |
| 39 | // Claim zoekt een aangeboden job voor runner rn en zet hem op running. |
| 40 | // nil, nil als er niets is. |
| 41 | func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) { |
| 42 | if r.storage() == "" { |
| 43 | return nil, nil |
| 44 | } |
| 45 | jobs, runs, err := r.Store.OfferedJobs(ctx) |
| 46 | if err != nil { |
| 47 | return nil, err |
| 48 | } |
| 49 | cache := map[int64][]Tier{} |
| 50 | for _, j := range jobs { |
| 51 | tiers := r.tiersFor(ctx, cache, j.RepoID) |
| 52 | if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) { |
| 53 | continue |
| 54 | } |
| 55 | repo, err := r.Store.RepoByID(ctx, j.RepoID) |
| 56 | if err != nil || !r.mayRun(ctx, rn, repo) { |
| 57 | continue |
| 58 | } |
| 59 | p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID) |
| 60 | if err != nil || (r.Hold != nil && r.Hold(p)) { |
| 61 | continue |
| 62 | } |
| 63 | if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy { |
| 64 | continue |
| 65 | } |
| 66 | ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name) |
| 67 | if err != nil || !ok { |
| 68 | continue |
| 69 | } |
| 70 | pl, err := r.payload(ctx, repo, p, j, rn) |
| 71 | if err != nil { |
| 72 | r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error()) |
| 73 | continue |
| 74 | } |
| 75 | return pl, nil |
| 76 | } |
| 77 | return nil, nil |
| 78 | } |
| 79 | |
| 80 | // mayRun is de privacycontrole: een account-runner krijgt alleen jobs |
| 81 | // uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit |
| 82 | // repo's van die groep. |
| 83 | func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool { |
| 84 | switch rn.OwnerKind { |
| 85 | case "group": |
| 86 | return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID |
| 87 | case "user": |
| 88 | if repo.OwnerKind == "user" { |
| 89 | return repo.OwnerID == rn.OwnerID |
| 90 | } |
| 91 | role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID) |
| 92 | return err == nil && role.CanRead() |
| 93 | } |
| 94 | return false |
| 95 | } |
| 96 | |
| 97 | func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) { |
| 98 | token, err := r.Store.CreateJobToken(ctx, job.ID) |
| 99 | if err != nil { |
| 100 | return nil, err |
| 101 | } |
| 102 | env, mask, err := r.environment(ctx, repo, p, job, token) |
| 103 | if err != nil { |
| 104 | return nil, err |
| 105 | } |
| 106 | remoteMasks.Store(job.ID, mask) |
| 107 | limit := r.Timeout |
| 108 | if job.Timeout > 0 { |
| 109 | limit = job.Timeout |
| 110 | } |
| 111 | script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID}) |
| 112 | return &RemotePayload{ |
| 113 | JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job, |
| 114 | CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git", |
| 115 | SHA: p.CommitSHA, Script: script, Env: env, Mask: mask, |
| 116 | Timeout: int64(limit / time.Second), |
| 117 | }, nil |
| 118 | } |
| 119 | |
| 120 | // RemoteLog voegt logregels van een aparte runner toe. cancel is true als |
| 121 | // de job niet meer loopt (geannuleerd of afgebroken). |
| 122 | func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) { |
| 123 | if job.Status != "running" { |
| 124 | return true, nil |
| 125 | } |
| 126 | if m, ok := remoteMasks.Load(job.ID); ok { |
| 127 | chunk = ci.Mask(chunk, m.([]string)) |
| 128 | } |
| 129 | if len(chunk) > 0 { |
| 130 | err = r.Store.AppendJobLog(ctx, job.ID, chunk) |
| 131 | } |
| 132 | return false, err |
| 133 | } |
| 134 | |
| 135 | // RemoteInputs zet de artifacts van eerdere jobs in dir. |
| 136 | func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error { |
| 137 | p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID) |
| 138 | if err != nil { |
| 139 | return err |
| 140 | } |
| 141 | return r.materialize(ctx, p, job, dir) |
| 142 | } |
| 143 | |
| 144 | // RemoteArtifacts bewaart de geüploade artifacts uit dir. |
| 145 | func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error { |
| 146 | return r.collect(ctx, job, dir) |
| 147 | } |
| 148 | |
| 149 | // RemoteFinish rondt een job van een aparte runner af. |
| 150 | func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error { |
| 151 | job, err := r.Store.JobForRunner(ctx, rn.ID, jobID) |
| 152 | if err != nil { |
| 153 | return err |
| 154 | } |
| 155 | if job.Status != "running" { |
| 156 | return nil |
| 157 | } |
| 158 | p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID) |
| 159 | if err != nil { |
| 160 | return err |
| 161 | } |
| 162 | var mask []string |
| 163 | if m, ok := remoteMasks.LoadAndDelete(job.ID); ok { |
| 164 | mask = m.([]string) |
| 165 | } |
| 166 | extra, kind := "", "" |
| 167 | switch status { |
| 168 | case "success": |
| 169 | case "canceled": |
| 170 | extra = stamp("canceled") |
| 171 | default: |
| 172 | status = "failed" |
| 173 | if message == "" { |
| 174 | message = "job failed" |
| 175 | } |
| 176 | kind = classifyFailure(errors.New(message)) |
| 177 | if strings.Contains(message, "job_execution_timeout") { |
| 178 | kind = "job_execution_timeout" |
| 179 | } |
| 180 | extra = stamp(ci.Mask(message, mask)) |
| 181 | } |
| 182 | r.complete(p, job, p, status, extra, kind) |
| 183 | return nil |
| 184 | } |
| 185 | |
| 186 | // reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn. |
| 187 | func (r *Runner) reapRemote(ctx context.Context) { |
| 188 | jobs, err := r.Store.RemoteRunningJobs(ctx) |
| 189 | if err != nil { |
| 190 | return |
| 191 | } |
| 192 | now := time.Now() |
| 193 | for _, rj := range jobs { |
| 194 | rn, err := r.Store.RunnerByID(ctx, rj.RunnerID) |
| 195 | msg := "" |
| 196 | switch { |
| 197 | case err != nil || rn.Revoked: |
| 198 | msg = "runner_system_failure: runner was removed" |
| 199 | case now.Sub(rn.LastSeen) > RemoteOnlineGrace: |
| 200 | msg = "runner_system_failure: runner " + rn.Name + " stopped responding" |
| 201 | default: |
| 202 | limit := r.Timeout |
| 203 | if rj.Job.Timeout > 0 { |
| 204 | limit = rj.Job.Timeout |
| 205 | } |
| 206 | if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace { |
| 207 | msg = "job_execution_timeout: no result from runner " + rn.Name |
| 208 | } |
| 209 | } |
| 210 | if msg != "" { |
| 211 | _ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg) |
| 212 | } |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | // RunRemote draait een job op de runnermachine: clone, cache, inputs, |
| 217 | // container, cache opslaan. De werkmap blijft staan voor de artifacts. |
| 218 | // clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs. |
| 219 | func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error { |
| 220 | sink := &logSink{out: out, secrets: pl.Mask} |
| 221 | sink.section("prepare", "Preparing the docker executor") |
| 222 | _ = os.RemoveAll(work) |
| 223 | if err := os.MkdirAll(work, 0o755); err != nil { |
| 224 | sink.closeSection() |
| 225 | return err |
| 226 | } |
| 227 | if err := clone(ctx, work); err != nil { |
| 228 | sink.closeSection() |
| 229 | return fmt.Errorf("clone: %w", err) |
| 230 | } |
| 231 | if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil { |
| 232 | sink.closeSection() |
| 233 | return fmt.Errorf("checkout %s: %w", pl.SHA, err) |
| 234 | } |
| 235 | if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil { |
| 236 | sink.closeSection() |
| 237 | return err |
| 238 | } |
| 239 | if err := inputs(ctx, work); err != nil { |
| 240 | sink.closeSection() |
| 241 | return fmt.Errorf("artifacts: %w", err) |
| 242 | } |
| 243 | if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil { |
| 244 | sink.closeSection() |
| 245 | return err |
| 246 | } |
| 247 | if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil { |
| 248 | return err |
| 249 | } |
| 250 | for _, rel := range pl.Job.ArtifactPaths { |
| 251 | if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() { |
| 252 | return fmt.Errorf("artifact %s ontbreekt", rel) |
| 253 | } |
| 254 | } |
| 255 | return r.saveCache(pl.RepoID, pl.Job, work) |
| 256 | } |
| 257 | |
| 258 | // WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz. |
| 259 | // Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee. |
| 260 | func WriteTar(w io.Writer, dir string, rels []string) error { |
| 261 | gz := gzip.NewWriter(w) |
| 262 | tw := tar.NewWriter(gz) |
| 263 | add := func(rel string) error { |
| 264 | path := filepath.Join(dir, filepath.FromSlash(rel)) |
| 265 | info, err := os.Stat(path) |
| 266 | if err != nil || !info.Mode().IsRegular() { |
| 267 | return nil |
| 268 | } |
| 269 | f, err := os.Open(path) |
| 270 | if err != nil { |
| 271 | return err |
| 272 | } |
| 273 | defer f.Close() |
| 274 | if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil { |
| 275 | return err |
| 276 | } |
| 277 | _, err = io.Copy(tw, f) |
| 278 | return err |
| 279 | } |
| 280 | if rels == nil { |
| 281 | err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { |
| 282 | if err != nil || info.IsDir() { |
| 283 | return err |
| 284 | } |
| 285 | rel, _ := filepath.Rel(dir, path) |
| 286 | return add(filepath.ToSlash(rel)) |
| 287 | }) |
| 288 | if err != nil { |
| 289 | return err |
| 290 | } |
| 291 | } else { |
| 292 | for _, rel := range rels { |
| 293 | if err := add(rel); err != nil { |
| 294 | return err |
| 295 | } |
| 296 | } |
| 297 | } |
| 298 | if err := tw.Close(); err != nil { |
| 299 | return err |
| 300 | } |
| 301 | return gz.Close() |
| 302 | } |
| 303 | |
| 304 | // ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir; |
| 305 | // met allowed niet-nil alleen die paden. max begrenst de totale grootte. |
| 306 | func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error { |
| 307 | gz, err := gzip.NewReader(rd) |
| 308 | if err != nil { |
| 309 | return err |
| 310 | } |
| 311 | tr := tar.NewReader(gz) |
| 312 | ok := map[string]bool{} |
| 313 | for _, a := range allowed { |
| 314 | ok[filepath.ToSlash(filepath.Clean(a))] = true |
| 315 | } |
| 316 | var total int64 |
| 317 | for { |
| 318 | h, err := tr.Next() |
| 319 | if errors.Is(err, io.EOF) { |
| 320 | return nil |
| 321 | } |
| 322 | if err != nil { |
| 323 | return err |
| 324 | } |
| 325 | if h.Typeflag != tar.TypeReg { |
| 326 | continue |
| 327 | } |
| 328 | name := filepath.ToSlash(filepath.Clean(h.Name)) |
| 329 | if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") { |
| 330 | return fmt.Errorf("unsafe path %q", h.Name) |
| 331 | } |
| 332 | if allowed != nil && !ok[name] { |
| 333 | continue |
| 334 | } |
| 335 | total += h.Size |
| 336 | if max > 0 && total > max { |
| 337 | return errors.New("archive too large") |
| 338 | } |
| 339 | dst := filepath.Join(dir, filepath.FromSlash(name)) |
| 340 | if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { |
| 341 | return err |
| 342 | } |
| 343 | f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) |
| 344 | if err != nil { |
| 345 | return err |
| 346 | } |
| 347 | if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil { |
| 348 | f.Close() |
| 349 | return err |
| 350 | } |
| 351 | f.Close() |
| 352 | } |
| 353 | } |