package runner import ( "archive/tar" "compress/gzip" "context" "errors" "fmt" "io" "os" "path/filepath" "strings" "sync" "time" "gitlab.com/Timfa/bluegit/internal/ci" "gitlab.com/Timfa/bluegit/internal/store" ) // RemotePayload is wat een aparte runner bij een job krijgt. type RemotePayload struct { JobID int64 `json:"job_id"` PipelineID int64 `json:"pipeline_id"` RepoID int64 `json:"repo_id"` Job store.Job `json:"job"` CloneURL string `json:"clone_url"` SHA string `json:"sha"` Script string `json:"script"` Env map[string]string `json:"env"` Mask []string `json:"mask"` Timeout int64 `json:"timeout_seconds"` } var remoteMasks sync.Map // job id → []string // RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job. var RemoteOnlineGrace = 5 * time.Minute // Claim zoekt een aangeboden job voor runner rn en zet hem op running. // nil, nil als er niets is. func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) { if r.storage() == "" { return nil, nil } jobs, runs, err := r.Store.OfferedJobs(ctx) if err != nil { return nil, err } cache := map[int64][]Tier{} for _, j := range jobs { tiers := r.tiersFor(ctx, cache, j.RepoID) if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) { continue } repo, err := r.Store.RepoByID(ctx, j.RepoID) if err != nil || !r.mayRun(ctx, rn, repo) { continue } p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID) if err != nil || (r.Hold != nil && r.Hold(p)) { continue } if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy { continue } ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name) if err != nil || !ok { continue } pl, err := r.payload(ctx, repo, p, j, rn) if err != nil { r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error()) continue } return pl, nil } return nil, nil } // mayRun is de privacycontrole: een account-runner krijgt alleen jobs // uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit // repo's van die groep. func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool { switch rn.OwnerKind { case "group": return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID case "user": if repo.OwnerKind == "user" { return repo.OwnerID == rn.OwnerID } role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID) return err == nil && role.CanRead() } return false } func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) { token, err := r.Store.CreateJobToken(ctx, job.ID) if err != nil { return nil, err } env, mask, err := r.environment(ctx, repo, p, job, token) if err != nil { return nil, err } remoteMasks.Store(job.ID, mask) limit := r.Timeout if job.Timeout > 0 { limit = job.Timeout } script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID}) return &RemotePayload{ JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job, CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git", SHA: p.CommitSHA, Script: script, Env: env, Mask: mask, Timeout: int64(limit / time.Second), }, nil } // RemoteLog voegt logregels van een aparte runner toe. cancel is true als // de job niet meer loopt (geannuleerd of afgebroken). func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) { if job.Status != "running" { return true, nil } if m, ok := remoteMasks.Load(job.ID); ok { chunk = ci.Mask(chunk, m.([]string)) } if len(chunk) > 0 { err = r.Store.AppendJobLog(ctx, job.ID, chunk) } return false, err } // RemoteInputs zet de artifacts van eerdere jobs in dir. func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error { p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID) if err != nil { return err } return r.materialize(ctx, p, job, dir) } // RemoteArtifacts bewaart de geüploade artifacts uit dir. func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error { return r.collect(ctx, job, dir) } // RemoteFinish rondt een job van een aparte runner af. func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error { job, err := r.Store.JobForRunner(ctx, rn.ID, jobID) if err != nil { return err } if job.Status != "running" { return nil } p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID) if err != nil { return err } var mask []string if m, ok := remoteMasks.LoadAndDelete(job.ID); ok { mask = m.([]string) } extra, kind := "", "" switch status { case "success": case "canceled": extra = stamp("canceled") default: status = "failed" if message == "" { message = "job failed" } kind = classifyFailure(errors.New(message)) if strings.Contains(message, "job_execution_timeout") { kind = "job_execution_timeout" } extra = stamp(ci.Mask(message, mask)) } r.complete(p, job, p, status, extra, kind) return nil } // reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn. func (r *Runner) reapRemote(ctx context.Context) { jobs, err := r.Store.RemoteRunningJobs(ctx) if err != nil { return } now := time.Now() for _, rj := range jobs { rn, err := r.Store.RunnerByID(ctx, rj.RunnerID) msg := "" switch { case err != nil || rn.Revoked: msg = "runner_system_failure: runner was removed" case now.Sub(rn.LastSeen) > RemoteOnlineGrace: msg = "runner_system_failure: runner " + rn.Name + " stopped responding" default: limit := r.Timeout if rj.Job.Timeout > 0 { limit = rj.Job.Timeout } if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace { msg = "job_execution_timeout: no result from runner " + rn.Name } } if msg != "" { _ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg) } } } // RunRemote draait een job op de runnermachine: clone, cache, inputs, // container, cache opslaan. De werkmap blijft staan voor de artifacts. // clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs. func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error { sink := &logSink{out: out, secrets: pl.Mask} sink.section("prepare", "Preparing the docker executor") _ = os.RemoveAll(work) if err := os.MkdirAll(work, 0o755); err != nil { sink.closeSection() return err } if err := clone(ctx, work); err != nil { sink.closeSection() return fmt.Errorf("clone: %w", err) } if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil { sink.closeSection() return fmt.Errorf("checkout %s: %w", pl.SHA, err) } if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil { sink.closeSection() return err } if err := inputs(ctx, work); err != nil { sink.closeSection() return fmt.Errorf("artifacts: %w", err) } if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil { sink.closeSection() return err } if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil { return err } for _, rel := range pl.Job.ArtifactPaths { if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() { return fmt.Errorf("artifact %s ontbreekt", rel) } } return r.saveCache(pl.RepoID, pl.Job, work) } // WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz. // Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee. func WriteTar(w io.Writer, dir string, rels []string) error { gz := gzip.NewWriter(w) tw := tar.NewWriter(gz) add := func(rel string) error { path := filepath.Join(dir, filepath.FromSlash(rel)) info, err := os.Stat(path) if err != nil || !info.Mode().IsRegular() { return nil } f, err := os.Open(path) if err != nil { return err } defer f.Close() if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil { return err } _, err = io.Copy(tw, f) return err } if rels == nil { err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } rel, _ := filepath.Rel(dir, path) return add(filepath.ToSlash(rel)) }) if err != nil { return err } } else { for _, rel := range rels { if err := add(rel); err != nil { return err } } } if err := tw.Close(); err != nil { return err } return gz.Close() } // ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir; // met allowed niet-nil alleen die paden. max begrenst de totale grootte. func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error { gz, err := gzip.NewReader(rd) if err != nil { return err } tr := tar.NewReader(gz) ok := map[string]bool{} for _, a := range allowed { ok[filepath.ToSlash(filepath.Clean(a))] = true } var total int64 for { h, err := tr.Next() if errors.Is(err, io.EOF) { return nil } if err != nil { return err } if h.Typeflag != tar.TypeReg { continue } name := filepath.ToSlash(filepath.Clean(h.Name)) if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") { return fmt.Errorf("unsafe path %q", h.Name) } if allowed != nil && !ok[name] { continue } total += h.Size if max > 0 && total > max { return errors.New("archive too large") } dst := filepath.Join(dir, filepath.FromSlash(name)) if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { return err } f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { return err } if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil { f.Close() return err } f.Close() } }