// Command bluegit-runner draait BlueGit CI-jobs op een eigen machine. // Hij maakt alleen uitgaande HTTPS-verbindingen (long-poll), er hoeft // geen poort open. package main import ( "bytes" "context" "encoding/json" "errors" "flag" "fmt" "io" "log/slog" "net/http" "os" "os/exec" "os/signal" "os/user" "path/filepath" "strconv" "strings" "sync" "syscall" "time" "gopkg.in/yaml.v3" "gitlab.com/Timfa/bluegit/internal/runner" "gitlab.com/Timfa/bluegit/internal/runnercfg" ) var version = "dev" const defaultConfig = "/etc/bluegit-runner/config.yml" // maxWorkers is de bovengrens voor concurrency. const maxWorkers = 64 // Config staat in /etc/bluegit-runner/config.yml. type Config struct { URL string `yaml:"url"` Token string `yaml:"token"` Name string `yaml:"name"` Docker string `yaml:"docker"` Git string `yaml:"git"` WorkDir string `yaml:"work_dir"` Network string `yaml:"network"` Concurrency int `yaml:"concurrency"` // UIListen is het adres van de webinterface; "off" zet hem uit. UIListen string `yaml:"ui_listen"` // Language is de standaardtaal van de webinterface (en, nl, de, fr). Language string `yaml:"language"` } func (c *Config) defaults() { if c.Docker == "" { c.Docker = "docker" } if c.Git == "" { c.Git = "git" } if c.WorkDir == "" { c.WorkDir = "/var/lib/bluegit-runner" } if c.Network == "" { c.Network = "bridge" } if c.Concurrency < 1 { c.Concurrency = 1 } if c.UIListen == "" { c.UIListen = defaultUIListen } c.URL = strings.TrimRight(c.URL, "/") } func main() { if len(os.Args) < 2 { usage() os.Exit(2) } var err error switch os.Args[1] { case "register": err = cmdRegister(os.Args[2:]) case "run": err = cmdRun(os.Args[2:]) case "ui-token": err = cmdUIToken(os.Args[2:]) case "version", "--version": fmt.Println(version) default: usage() os.Exit(2) } if err != nil { fmt.Fprintln(os.Stderr, "bluegit-runner:", err) os.Exit(1) } } func usage() { fmt.Fprintln(os.Stderr, `usage: bluegit-runner register --url https://git.example.com --token bgrr_... [--name NAME] [--config PATH] bluegit-runner run [--config PATH] bluegit-runner ui-token [--config PATH] (shows the web UI admin token) bluegit-runner version`) } func loadConfig(path string) (Config, error) { var c Config b, err := os.ReadFile(path) if err != nil { return c, err } if err := yaml.Unmarshal(b, &c); err != nil { return c, err } c.defaults() if c.URL == "" || c.Token == "" { return c, fmt.Errorf("%s: url and token are required; run bluegit-runner register first", path) } return c, nil } func cmdRegister(args []string) error { fs := flag.NewFlagSet("register", flag.ExitOnError) url := fs.String("url", "", "BlueGit base URL") token := fs.String("token", "", "registration token from the BlueGit UI") name := fs.String("name", "", "runner name (default: hostname)") cfgPath := fs.String("config", defaultConfig, "config file") _ = fs.Parse(args) // Niet-interactief (agents): alles mag ook via de omgeving. if *token == "" { *token = os.Getenv("BLUEGIT_RUNNER_REGISTRATION_TOKEN") } if *url == "" { *url = os.Getenv("BLUEGIT_RUNNER_URL") } if *name == "" { *name = os.Getenv("BLUEGIT_RUNNER_NAME") } if *url == "" || *token == "" { return errors.New("--url and --token are required") } if *name == "" { *name, _ = os.Hostname() } var existing Config if b, err := os.ReadFile(*cfgPath); err == nil { _ = yaml.Unmarshal(b, &existing) } body, _ := json.Marshal(map[string]string{"token": *token, "name": *name, "version": version}) resp, err := http.Post(strings.TrimRight(*url, "/")+"/api/v1/runners/register", "application/json", bytes.NewReader(body)) if err != nil { return err } defer resp.Body.Close() var out struct { ID int64 `json:"id"` Name string `json:"name"` Token string `json:"token"` Message string `json:"message"` } _ = json.NewDecoder(resp.Body).Decode(&out) if resp.StatusCode != http.StatusCreated || out.Token == "" { return fmt.Errorf("registration failed (%d): %s", resp.StatusCode, out.Message) } existing.URL, existing.Token, existing.Name = strings.TrimRight(*url, "/"), out.Token, out.Name b, err := yaml.Marshal(existing) if err != nil { return err } if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil { return err } if err := os.WriteFile(*cfgPath, b, 0o600); err != nil { return err } if os.Geteuid() == 0 { if u, err := user.Lookup("bluegit-runner"); err == nil { uid, _ := strconv.Atoi(u.Uid) gid, _ := strconv.Atoi(u.Gid) _ = os.Chown(*cfgPath, uid, gid) } } fmt.Printf("registered runner %q (id %d); config written to %s\n", out.Name, out.ID, *cfgPath) fmt.Printf("web UI: http://%s (admin token: sudo bluegit-runner ui-token)\n", defaultUIListen) // Met het AUR-pakket: de service is al enabled en wacht op de config. if os.Geteuid() == 0 && *cfgPath == defaultConfig { if _, err := os.Stat("/usr/lib/systemd/system/bluegit-runner.service"); err == nil { if err := exec.Command("systemctl", "enable", "--now", "bluegit-runner.service").Run(); err == nil { _ = exec.Command("systemctl", "restart", "bluegit-runner.service").Run() fmt.Println("bluegit-runner.service started") return nil } } } fmt.Println("start it with: sudo systemctl enable --now bluegit-runner") return nil } type client struct { cfg Config http *http.Client log *slog.Logger // beheer (webinterface) file *runnercfg.File jobs *runnercfg.Tracker stMu sync.Mutex seen time.Time srvName string srvPause bool } func (c *client) settings() runnercfg.Settings { s, err := c.file.Load(c.defaults()) if err != nil { return c.defaults() } return s } func (c *client) defaults() runnercfg.Settings { d := runnercfg.Defaults() d.Concurrency = c.cfg.Concurrency return d } func (c *client) status() runnercfg.Status { c.stMu.Lock() defer c.stMu.Unlock() name := c.settings().Name if name == "" { name = c.srvName } if name == "" { name = c.cfg.Name } return runnercfg.Status{Kind: "standalone", Name: name, Version: version, ServerURL: c.cfg.URL, Online: !c.seen.IsZero() && time.Since(c.seen) < 90*time.Second, LastContact: c.seen, ServerPause: c.srvPause} } // heartbeat meldt pauze en (lokaal ingestelde) naam aan de server. func (c *client) heartbeat(ctx context.Context) error { s := c.settings() var out struct { Name string `json:"name"` Paused bool `json:"paused"` } err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]any{"draining": s.Paused, "name": s.Name}, &out) if err == nil { c.stMu.Lock() c.seen, c.srvName, c.srvPause = time.Now(), out.Name, out.Paused c.stMu.Unlock() } return err } func cmdUIToken(args []string) error { fs := flag.NewFlagSet("ui-token", flag.ExitOnError) cfgPath := fs.String("config", defaultConfig, "config file") _ = fs.Parse(args) b, err := os.ReadFile(*cfgPath) if err != nil { return err } var cfg Config if err := yaml.Unmarshal(b, &cfg); err != nil { return err } cfg.defaults() tok, _, err := ensureUIToken(cfg) if err != nil { return fmt.Errorf("%w (run as root or as bluegit-runner)", err) } fmt.Printf("web UI: http://%s\nadmin token: %s\n", cfg.UIListen, tok) return nil } func (c *client) do(ctx context.Context, method, path string, body io.Reader, ctype string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, method, c.cfg.URL+path, body) if err != nil { return nil, err } req.Header.Set("Authorization", "Bearer "+c.cfg.Token) req.Header.Set("X-Runner-Version", version) if ctype != "" { req.Header.Set("Content-Type", ctype) } return c.http.Do(req) } func (c *client) postJSON(ctx context.Context, path string, in, out any) error { b, _ := json.Marshal(in) resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b), "application/json") if err != nil { return err } defer resp.Body.Close() if resp.StatusCode >= 300 { msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) return fmt.Errorf("%s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(msg))) } if out != nil { return json.NewDecoder(resp.Body).Decode(out) } return nil } func cmdRun(args []string) error { fs := flag.NewFlagSet("run", flag.ExitOnError) cfgPath := fs.String("config", defaultConfig, "config file") once := fs.Bool("once", false, "run at most one job, then exit (for tests)") _ = fs.Parse(args) cfg, err := loadConfig(*cfgPath) if err != nil { return err } log := slog.New(slog.NewTextHandler(os.Stderr, nil)) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() c := &client{cfg: cfg, http: &http.Client{Timeout: 0}, log: log, file: &runnercfg.File{Path: filepath.Join(cfg.WorkDir, "settings.json")}, jobs: &runnercfg.Tracker{}} c.jobs.SetPaused(c.settings().Paused) if err := c.heartbeat(ctx); err != nil { return fmt.Errorf("cannot reach BlueGit: %w", err) } log.Info("bluegit-runner started", "url", cfg.URL, "name", c.status().Name, "version", version, "concurrency", c.settings().Concurrency) if cfg.UIListen != "off" { tok, created, err := ensureUIToken(cfg) if err != nil { log.Warn("web UI token", "err", err) } else { h := &runnercfg.Handler{File: c.file, Default: c.defaults(), Tracker: c.jobs, WorkDir: cfg.WorkDir, Status: c.status, Apply: func(runnercfg.Settings) error { hctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = c.heartbeat(hctx) // naam en pauze meteen naar de server return nil }} if err := serveUI(ctx, newWebUI(cfg, tok, h), cfg.UIListen, log.Warn); err != nil { log.Warn("web UI", "listen", cfg.UIListen, "err", err) } else { log.Info("web UI", "url", "http://"+cfg.UIListen, "first_start", created, "token", "see: bluegit-runner ui-token") } } } go func() { t := time.NewTicker(30 * time.Second) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: if err := c.heartbeat(ctx); err != nil && ctx.Err() == nil { log.Warn("heartbeat", "err", err) } } } }() var wg sync.WaitGroup // Een vaste pool; worker i werkt alleen als i < concurrency en niet gepauzeerd. for i := 0; i < maxWorkers; i++ { wg.Add(1) go func(i int) { defer wg.Done() for ctx.Err() == nil { s := c.settings() if s.Paused || i >= s.Concurrency { select { case <-ctx.Done(): case <-time.After(2 * time.Second): } continue } ran, err := c.poll(ctx) if err != nil && ctx.Err() == nil { log.Warn("poll", "err", err) select { case <-ctx.Done(): case <-time.After(5 * time.Second): } } if ran && *once { stop() } } }(i) } wg.Wait() return nil } func (c *client) poll(ctx context.Context) (bool, error) { pctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() resp, err := c.do(pctx, http.MethodPost, "/api/v1/runner/jobs/request", strings.NewReader("{}"), "application/json") if err != nil { return false, err } defer resp.Body.Close() if resp.StatusCode == http.StatusNoContent { return false, nil } if resp.StatusCode != http.StatusOK { msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) return false, fmt.Errorf("request: %d %s", resp.StatusCode, strings.TrimSpace(string(msg))) } var pl runner.RemotePayload if err := json.NewDecoder(resp.Body).Decode(&pl); err != nil { return false, err } c.runJob(ctx, pl) return true, nil } // logBuffer verzamelt logregels en stuurt ze elke seconde door. type logBuffer struct { mu sync.Mutex buf strings.Builder } func (l *logBuffer) add(s string) { l.mu.Lock() l.buf.WriteString(s) l.mu.Unlock() } func (l *logBuffer) take() string { l.mu.Lock() defer l.mu.Unlock() s := l.buf.String() l.buf.Reset() return s } func (c *client) runJob(parent context.Context, pl runner.RemotePayload) { c.log.Info("job", "id", pl.JobID, "name", pl.Job.Name) base := fmt.Sprintf("/api/v1/runner/jobs/%d", pl.JobID) ctx, cancel := context.WithCancel(parent) defer cancel() runCtx := ctx if m := c.settings().JobTimeoutMin; m > 0 && (pl.Timeout <= 0 || int64(m*60) < pl.Timeout) { pl.Timeout = int64(m * 60) } if pl.Timeout > 0 { var tc context.CancelFunc runCtx, tc = context.WithTimeout(ctx, time.Duration(pl.Timeout)*time.Second) defer tc() } var canceled bool var cmu sync.Mutex logs := &logBuffer{} send := func() { chunk := logs.take() var out struct { Cancel bool `json:"cancel"` } sctx, sc := context.WithTimeout(context.Background(), 30*time.Second) defer sc() err := func() error { resp, err := c.do(sctx, http.MethodPost, base+"/log", strings.NewReader(chunk), "text/plain") if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("log: %d", resp.StatusCode) } return json.NewDecoder(resp.Body).Decode(&out) }() if err != nil { // Bij een fout gaat de chunk terug in de buffer voor de volgende poging. if chunk != "" { logs.mu.Lock() rest := logs.buf.String() logs.buf.Reset() logs.buf.WriteString(chunk + rest) logs.mu.Unlock() } return } if out.Cancel { cmu.Lock() canceled = true cmu.Unlock() cancel() } } done := make(chan struct{}) flushed := make(chan struct{}) go func() { defer close(flushed) t := time.NewTicker(time.Second) defer t.Stop() for { select { case <-done: return case <-t.C: send() } } }() st := c.settings() r := &runner.Runner{GitBin: c.cfg.Git, Docker: c.cfg.Docker, Storage: c.cfg.WorkDir, Network: c.cfg.Network, Name: c.status().Name} if over, oerr := st.Overrides(c.cfg.WorkDir, nil); oerr == nil { r.Configure(st, over) } c.jobs.Start(runnercfg.JobInfo{ID: pl.JobID, Name: pl.Job.Name, Pipeline: pl.PipelineID, Image: pl.Job.Image, Project: pl.Env["CI_PROJECT_PATH"], URL: pl.Env["CI_JOB_URL"]}) work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10)) defer func() { if st.CleanupWork { _ = runner.ClearWorkDir(work) } runnercfg.Cleanup(context.Background(), c.cfg.Docker, c.settings(), c.jobs, filepath.Join(c.cfg.WorkDir, "cache")) }() clone := func(ctx context.Context, dir string) error { cmd := exec.CommandContext(ctx, c.cfg.Git, "clone", "--quiet", pl.CloneURL, dir) // Het runner-token via de omgeving, niet in de proceslijst. cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=Authorization: Bearer "+c.cfg.Token) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("%s", strings.ReplaceAll(strings.TrimSpace(string(out)), c.cfg.Token, "[MASKED]")) } return nil } inputs := func(ctx context.Context, dir string) error { resp, err := c.do(ctx, http.MethodGet, base+"/inputs", nil, "") if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("inputs: %d", resp.StatusCode) } return runner.ReadTar(resp.Body, dir, nil, 0) } err := r.RunRemote(runCtx, pl, work, logs.add, clone, inputs) if err == nil && len(pl.Job.ArtifactPaths) > 0 { err = c.upload(runCtx, base, work, pl.Job.ArtifactPaths) } close(done) <-flushed send() status, msg := "success", "" cmu.Lock() wasCanceled := canceled cmu.Unlock() switch { case err == nil: case wasCanceled: status = "canceled" case errors.Is(runCtx.Err(), context.DeadlineExceeded): status, msg = "failed", "job_execution_timeout: "+err.Error() default: status, msg = "failed", err.Error() } fctx, fc := context.WithTimeout(context.Background(), time.Minute) defer fc() if ferr := c.postJSON(fctx, base+"/finish", map[string]string{"status": status, "message": msg}, nil); ferr != nil { c.log.Error("finish", "job", pl.JobID, "err", ferr) } c.jobs.Finish(pl.JobID, status) c.log.Info("job done", "id", pl.JobID, "status", status) } func (c *client) upload(ctx context.Context, base, work string, paths []string) error { pr, pw := io.Pipe() go func() { pw.CloseWithError(runner.WriteTar(pw, work, paths)) }() resp, err := c.do(ctx, http.MethodPut, base+"/artifacts", pr, "application/gzip") if err != nil { return fmt.Errorf("artifacts upload: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) return fmt.Errorf("artifacts upload: %d %s", resp.StatusCode, strings.TrimSpace(string(msg))) } return nil }