BlueGit

timfa bluegit

Files

bluegit / cmd / bluegit-runner / main.go

main.go

main.go 16.8 KiB Download Raw
1// Command bluegit-runner draait BlueGit CI-jobs op een eigen machine.
2// Hij maakt alleen uitgaande HTTPS-verbindingen (long-poll), er hoeft
3// geen poort open.
4package main
5
6import (
7 "bytes"
8 "context"
9 "encoding/json"
10 "errors"
11 "flag"
12 "fmt"
13 "io"
14 "log/slog"
15 "net/http"
16 "os"
17 "os/exec"
18 "os/signal"
19 "os/user"
20 "path/filepath"
21 "strconv"
22 "strings"
23 "sync"
24 "syscall"
25 "time"
26
27 "gopkg.in/yaml.v3"
28
29 "gitlab.com/Timfa/bluegit/internal/runner"
30 "gitlab.com/Timfa/bluegit/internal/runnercfg"
31)
32
33var version = "dev"
34
35const defaultConfig = "/etc/bluegit-runner/config.yml"
36
37// maxWorkers is de bovengrens voor concurrency.
38const maxWorkers = 64
39
40// Config staat in /etc/bluegit-runner/config.yml.
41type Config struct {
42 URL string `yaml:"url"`
43 Token string `yaml:"token"`
44 Name string `yaml:"name"`
45 Docker string `yaml:"docker"`
46 Git string `yaml:"git"`
47 WorkDir string `yaml:"work_dir"`
48 Network string `yaml:"network"`
49 Concurrency int `yaml:"concurrency"`
50 // UIListen is het adres van de webinterface; "off" zet hem uit.
51 UIListen string `yaml:"ui_listen"`
52 // Language is de standaardtaal van de webinterface (en, nl, de, fr).
53 Language string `yaml:"language"`
54}
55
56func (c *Config) defaults() {
57 if c.Docker == "" {
58 c.Docker = "docker"
59 }
60 if c.Git == "" {
61 c.Git = "git"
62 }
63 if c.WorkDir == "" {
64 c.WorkDir = "/var/lib/bluegit-runner"
65 }
66 if c.Network == "" {
67 c.Network = "bridge"
68 }
69 if c.Concurrency < 1 {
70 c.Concurrency = 1
71 }
72 if c.UIListen == "" {
73 c.UIListen = defaultUIListen
74 }
75 c.URL = strings.TrimRight(c.URL, "/")
76}
77
78func main() {
79 if len(os.Args) < 2 {
80 usage()
81 os.Exit(2)
82 }
83 var err error
84 switch os.Args[1] {
85 case "register":
86 err = cmdRegister(os.Args[2:])
87 case "run":
88 err = cmdRun(os.Args[2:])
89 case "ui-token":
90 err = cmdUIToken(os.Args[2:])
91 case "version", "--version":
92 fmt.Println(version)
93 default:
94 usage()
95 os.Exit(2)
96 }
97 if err != nil {
98 fmt.Fprintln(os.Stderr, "bluegit-runner:", err)
99 os.Exit(1)
100 }
101}
102
103func usage() {
104 fmt.Fprintln(os.Stderr, `usage:
105 bluegit-runner register --url https://git.example.com --token bgrr_... [--name NAME] [--config PATH]
106 bluegit-runner run [--config PATH]
107 bluegit-runner ui-token [--config PATH] (shows the web UI admin token)
108 bluegit-runner version`)
109}
110
111func loadConfig(path string) (Config, error) {
112 var c Config
113 b, err := os.ReadFile(path)
114 if err != nil {
115 return c, err
116 }
117 if err := yaml.Unmarshal(b, &c); err != nil {
118 return c, err
119 }
120 c.defaults()
121 if c.URL == "" || c.Token == "" {
122 return c, fmt.Errorf("%s: url and token are required; run bluegit-runner register first", path)
123 }
124 return c, nil
125}
126
127func cmdRegister(args []string) error {
128 fs := flag.NewFlagSet("register", flag.ExitOnError)
129 url := fs.String("url", "", "BlueGit base URL")
130 token := fs.String("token", "", "registration token from the BlueGit UI")
131 name := fs.String("name", "", "runner name (default: hostname)")
132 cfgPath := fs.String("config", defaultConfig, "config file")
133 _ = fs.Parse(args)
134 // Niet-interactief (agents): alles mag ook via de omgeving.
135 if *token == "" {
136 *token = os.Getenv("BLUEGIT_RUNNER_REGISTRATION_TOKEN")
137 }
138 if *url == "" {
139 *url = os.Getenv("BLUEGIT_RUNNER_URL")
140 }
141 if *name == "" {
142 *name = os.Getenv("BLUEGIT_RUNNER_NAME")
143 }
144 if *url == "" || *token == "" {
145 return errors.New("--url and --token are required")
146 }
147 if *name == "" {
148 *name, _ = os.Hostname()
149 }
150 var existing Config
151 if b, err := os.ReadFile(*cfgPath); err == nil {
152 _ = yaml.Unmarshal(b, &existing)
153 }
154 body, _ := json.Marshal(map[string]string{"token": *token, "name": *name, "version": version})
155 resp, err := http.Post(strings.TrimRight(*url, "/")+"/api/v1/runners/register", "application/json", bytes.NewReader(body))
156 if err != nil {
157 return err
158 }
159 defer resp.Body.Close()
160 var out struct {
161 ID int64 `json:"id"`
162 Name string `json:"name"`
163 Token string `json:"token"`
164 Message string `json:"message"`
165 }
166 _ = json.NewDecoder(resp.Body).Decode(&out)
167 if resp.StatusCode != http.StatusCreated || out.Token == "" {
168 return fmt.Errorf("registration failed (%d): %s", resp.StatusCode, out.Message)
169 }
170 existing.URL, existing.Token, existing.Name = strings.TrimRight(*url, "/"), out.Token, out.Name
171 b, err := yaml.Marshal(existing)
172 if err != nil {
173 return err
174 }
175 if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil {
176 return err
177 }
178 if err := os.WriteFile(*cfgPath, b, 0o600); err != nil {
179 return err
180 }
181 if os.Geteuid() == 0 {
182 if u, err := user.Lookup("bluegit-runner"); err == nil {
183 uid, _ := strconv.Atoi(u.Uid)
184 gid, _ := strconv.Atoi(u.Gid)
185 _ = os.Chown(*cfgPath, uid, gid)
186 }
187 }
188 fmt.Printf("registered runner %q (id %d); config written to %s\n", out.Name, out.ID, *cfgPath)
189 fmt.Printf("web UI: http://%s (admin token: sudo bluegit-runner ui-token)\n", defaultUIListen)
190 // Met het AUR-pakket: de service is al enabled en wacht op de config.
191 if os.Geteuid() == 0 && *cfgPath == defaultConfig {
192 if _, err := os.Stat("/usr/lib/systemd/system/bluegit-runner.service"); err == nil {
193 if err := exec.Command("systemctl", "enable", "--now", "bluegit-runner.service").Run(); err == nil {
194 _ = exec.Command("systemctl", "restart", "bluegit-runner.service").Run()
195 fmt.Println("bluegit-runner.service started")
196 return nil
197 }
198 }
199 }
200 fmt.Println("start it with: sudo systemctl enable --now bluegit-runner")
201 return nil
202}
203
204type client struct {
205 cfg Config
206 http *http.Client
207 log *slog.Logger
208 // beheer (webinterface)
209 file *runnercfg.File
210 jobs *runnercfg.Tracker
211 stMu sync.Mutex
212 seen time.Time
213 srvName string
214 srvPause bool
215}
216
217func (c *client) settings() runnercfg.Settings {
218 s, err := c.file.Load(c.defaults())
219 if err != nil {
220 return c.defaults()
221 }
222 return s
223}
224
225func (c *client) defaults() runnercfg.Settings {
226 d := runnercfg.Defaults()
227 d.Concurrency = c.cfg.Concurrency
228 return d
229}
230
231func (c *client) status() runnercfg.Status {
232 c.stMu.Lock()
233 defer c.stMu.Unlock()
234 name := c.settings().Name
235 if name == "" {
236 name = c.srvName
237 }
238 if name == "" {
239 name = c.cfg.Name
240 }
241 docker := runnercfg.PingDocker(context.Background(), c.cfg.Docker)
242 online := !c.seen.IsZero() && time.Since(c.seen) < 90*time.Second && docker.Reachable
243 return runnercfg.Status{Kind: "standalone", Name: name, Version: version, ServerURL: c.cfg.URL,
244 Online: online, LastContact: c.seen, ServerPause: c.srvPause, Docker: docker}
245}
246
247// heartbeat meldt pauze en (lokaal ingestelde) naam aan de server.
248func (c *client) heartbeat(ctx context.Context) error {
249 s := c.settings()
250 var out struct {
251 Name string `json:"name"`
252 Paused bool `json:"paused"`
253 }
254 err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]any{"draining": s.Paused, "name": s.Name}, &out)
255 if err == nil {
256 c.stMu.Lock()
257 c.seen, c.srvName, c.srvPause = time.Now(), out.Name, out.Paused
258 c.stMu.Unlock()
259 }
260 return err
261}
262
263func cmdUIToken(args []string) error {
264 fs := flag.NewFlagSet("ui-token", flag.ExitOnError)
265 cfgPath := fs.String("config", defaultConfig, "config file")
266 _ = fs.Parse(args)
267 b, err := os.ReadFile(*cfgPath)
268 if err != nil {
269 return err
270 }
271 var cfg Config
272 if err := yaml.Unmarshal(b, &cfg); err != nil {
273 return err
274 }
275 cfg.defaults()
276 tok, _, err := ensureUIToken(cfg)
277 if err != nil {
278 return fmt.Errorf("%w (run as root or as bluegit-runner)", err)
279 }
280 fmt.Printf("web UI: http://%s\nadmin token: %s\n", cfg.UIListen, tok)
281 return nil
282}
283
284func (c *client) do(ctx context.Context, method, path string, body io.Reader, ctype string) (*http.Response, error) {
285 req, err := http.NewRequestWithContext(ctx, method, c.cfg.URL+path, body)
286 if err != nil {
287 return nil, err
288 }
289 req.Header.Set("Authorization", "Bearer "+c.cfg.Token)
290 req.Header.Set("X-Runner-Version", version)
291 if ctype != "" {
292 req.Header.Set("Content-Type", ctype)
293 }
294 return c.http.Do(req)
295}
296
297func (c *client) postJSON(ctx context.Context, path string, in, out any) error {
298 b, _ := json.Marshal(in)
299 resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b), "application/json")
300 if err != nil {
301 return err
302 }
303 defer resp.Body.Close()
304 if resp.StatusCode >= 300 {
305 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
306 return fmt.Errorf("%s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(msg)))
307 }
308 if out != nil {
309 return json.NewDecoder(resp.Body).Decode(out)
310 }
311 return nil
312}
313
314func cmdRun(args []string) error {
315 fs := flag.NewFlagSet("run", flag.ExitOnError)
316 cfgPath := fs.String("config", defaultConfig, "config file")
317 once := fs.Bool("once", false, "run at most one job, then exit (for tests)")
318 _ = fs.Parse(args)
319 cfg, err := loadConfig(*cfgPath)
320 if err != nil {
321 return err
322 }
323 log := slog.New(slog.NewTextHandler(os.Stderr, nil))
324 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
325 defer stop()
326 c := &client{cfg: cfg, http: &http.Client{Timeout: 0}, log: log,
327 file: &runnercfg.File{Path: filepath.Join(cfg.WorkDir, "settings.json")}, jobs: &runnercfg.Tracker{}}
328 c.jobs.SetPaused(c.settings().Paused)
329 if err := c.heartbeat(ctx); err != nil {
330 return fmt.Errorf("cannot reach BlueGit: %w", err)
331 }
332 log.Info("bluegit-runner started", "url", cfg.URL, "name", c.status().Name, "version", version, "concurrency", c.settings().Concurrency)
333 if cfg.UIListen != "off" {
334 tok, created, err := ensureUIToken(cfg)
335 if err != nil {
336 log.Warn("web UI token", "err", err)
337 } else {
338 h := &runnercfg.Handler{File: c.file, Default: c.defaults(), Tracker: c.jobs, WorkDir: cfg.WorkDir,
339 Status: c.status, Apply: func(runnercfg.Settings) error {
340 hctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
341 defer cancel()
342 _ = c.heartbeat(hctx) // naam en pauze meteen naar de server
343 return nil
344 }}
345 if err := serveUI(ctx, newWebUI(cfg, tok, h), cfg.UIListen, log.Warn); err != nil {
346 log.Warn("web UI", "listen", cfg.UIListen, "err", err)
347 } else {
348 log.Info("web UI", "url", "http://"+cfg.UIListen, "first_start", created, "token", "see: bluegit-runner ui-token")
349 }
350 }
351 }
352 go func() {
353 t := time.NewTicker(30 * time.Second)
354 defer t.Stop()
355 for {
356 select {
357 case <-ctx.Done():
358 return
359 case <-t.C:
360 if err := c.heartbeat(ctx); err != nil && ctx.Err() == nil {
361 log.Warn("heartbeat", "err", err)
362 }
363 }
364 }
365 }()
366 var wg sync.WaitGroup
367 // Een vaste pool; worker i werkt alleen als i < concurrency en niet gepauzeerd.
368 for i := 0; i < maxWorkers; i++ {
369 wg.Add(1)
370 go func(i int) {
371 defer wg.Done()
372 for ctx.Err() == nil {
373 s := c.settings()
374 if s.Paused || i >= s.Concurrency {
375 select {
376 case <-ctx.Done():
377 case <-time.After(2 * time.Second):
378 }
379 continue
380 }
381 ran, err := c.poll(ctx)
382 if err != nil && ctx.Err() == nil {
383 log.Warn("poll", "err", err)
384 select {
385 case <-ctx.Done():
386 case <-time.After(5 * time.Second):
387 }
388 }
389 if ran && *once {
390 stop()
391 }
392 }
393 }(i)
394 }
395 wg.Wait()
396 return nil
397}
398
399func (c *client) poll(ctx context.Context) (bool, error) {
400 pctx, cancel := context.WithTimeout(ctx, 60*time.Second)
401 defer cancel()
402 resp, err := c.do(pctx, http.MethodPost, "/api/v1/runner/jobs/request", strings.NewReader("{}"), "application/json")
403 if err != nil {
404 return false, err
405 }
406 defer resp.Body.Close()
407 if resp.StatusCode == http.StatusNoContent {
408 return false, nil
409 }
410 if resp.StatusCode != http.StatusOK {
411 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
412 return false, fmt.Errorf("request: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
413 }
414 var pl runner.RemotePayload
415 if err := json.NewDecoder(resp.Body).Decode(&pl); err != nil {
416 return false, err
417 }
418 c.runJob(ctx, pl)
419 return true, nil
420}
421
422// logBuffer verzamelt logregels en stuurt ze elke seconde door.
423type logBuffer struct {
424 mu sync.Mutex
425 buf strings.Builder
426}
427
428func (l *logBuffer) add(s string) {
429 l.mu.Lock()
430 l.buf.WriteString(s)
431 l.mu.Unlock()
432}
433
434func (l *logBuffer) take() string {
435 l.mu.Lock()
436 defer l.mu.Unlock()
437 s := l.buf.String()
438 l.buf.Reset()
439 return s
440}
441
442func (c *client) runJob(parent context.Context, pl runner.RemotePayload) {
443 c.log.Info("job", "id", pl.JobID, "name", pl.Job.Name)
444 base := fmt.Sprintf("/api/v1/runner/jobs/%d", pl.JobID)
445 if pl.Job.DoNotInterrupt {
446 // Stoppen van de runner (SIGTERM) breekt deze job niet af; de
447 // workers wachten tot hij klaar is. Annuleren met force werkt wel.
448 parent = context.WithoutCancel(parent)
449 }
450 ctx, cancel := context.WithCancel(parent)
451 defer cancel()
452 runCtx := ctx
453 limitSource := runner.TimeoutFromServer
454 if pl.Job.Timeout > 0 {
455 limitSource = runner.TimeoutFromJob
456 }
457 if m := c.settings().JobTimeoutMin; m > 0 && (pl.Timeout <= 0 || int64(m*60) < pl.Timeout) {
458 pl.Timeout = int64(m * 60)
459 limitSource = runner.TimeoutFromRunner
460 }
461 if pl.Timeout > 0 {
462 var tc context.CancelFunc
463 runCtx, tc = context.WithTimeout(ctx, time.Duration(pl.Timeout)*time.Second)
464 defer tc()
465 }
466 var canceled bool
467 var cmu sync.Mutex
468 logs := &logBuffer{}
469 send := func() {
470 chunk := logs.take()
471 var out struct {
472 Cancel bool `json:"cancel"`
473 }
474 sctx, sc := context.WithTimeout(context.Background(), 30*time.Second)
475 defer sc()
476 err := func() error {
477 resp, err := c.do(sctx, http.MethodPost, base+"/log", strings.NewReader(chunk), "text/plain")
478 if err != nil {
479 return err
480 }
481 defer resp.Body.Close()
482 if resp.StatusCode != http.StatusOK {
483 return fmt.Errorf("log: %d", resp.StatusCode)
484 }
485 return json.NewDecoder(resp.Body).Decode(&out)
486 }()
487 if err != nil {
488 // Bij een fout gaat de chunk terug in de buffer voor de volgende poging.
489 if chunk != "" {
490 logs.mu.Lock()
491 rest := logs.buf.String()
492 logs.buf.Reset()
493 logs.buf.WriteString(chunk + rest)
494 logs.mu.Unlock()
495 }
496 return
497 }
498 if out.Cancel {
499 cmu.Lock()
500 canceled = true
501 cmu.Unlock()
502 cancel()
503 }
504 }
505 done := make(chan struct{})
506 flushed := make(chan struct{})
507 go func() {
508 defer close(flushed)
509 t := time.NewTicker(time.Second)
510 defer t.Stop()
511 for {
512 select {
513 case <-done:
514 return
515 case <-t.C:
516 send()
517 }
518 }
519 }()
520 st := c.settings()
521 r := &runner.Runner{GitBin: c.cfg.Git, Docker: c.cfg.Docker, Storage: c.cfg.WorkDir, Network: c.cfg.Network, Name: c.status().Name}
522 if over, oerr := st.Overrides(c.cfg.WorkDir, nil); oerr == nil {
523 r.Configure(st, over)
524 }
525 c.jobs.Start(runnercfg.JobInfo{ID: pl.JobID, Name: pl.Job.Name, Pipeline: pl.PipelineID, Image: pl.Job.Image,
526 Project: pl.Env["CI_PROJECT_PATH"], URL: pl.Env["CI_JOB_URL"]})
527 work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))
528 defer func() {
529 if st.CleanupWork {
530 _ = runner.ClearWorkDir(work)
531 }
532 runnercfg.Cleanup(context.Background(), c.cfg.Docker, c.settings(), c.jobs, filepath.Join(c.cfg.WorkDir, "cache"))
533 }()
534 clone := func(ctx context.Context, dir string) error {
535 cmd := exec.CommandContext(ctx, c.cfg.Git, "clone", "--quiet", pl.CloneURL, dir)
536 // Het runner-token via de omgeving, niet in de proceslijst.
537 cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_COUNT=1",
538 "GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=Authorization: Bearer "+c.cfg.Token)
539 out, err := cmd.CombinedOutput()
540 if err != nil {
541 return fmt.Errorf("%s", strings.ReplaceAll(strings.TrimSpace(string(out)), c.cfg.Token, "[MASKED]"))
542 }
543 return nil
544 }
545 inputs := func(ctx context.Context, dir string) error {
546 resp, err := c.do(ctx, http.MethodGet, base+"/inputs", nil, "")
547 if err != nil {
548 return err
549 }
550 defer resp.Body.Close()
551 if resp.StatusCode != http.StatusOK {
552 return fmt.Errorf("inputs: %d", resp.StatusCode)
553 }
554 return runner.ReadTar(resp.Body, dir, nil, 0)
555 }
556 err := r.RunRemote(runCtx, pl, work, logs.add, clone, inputs)
557 if err == nil && len(pl.Job.ArtifactPaths) > 0 {
558 err = c.upload(runCtx, base, work, pl.Job.ArtifactPaths)
559 }
560 close(done)
561 <-flushed
562 send()
563 status, msg := "success", ""
564 cmu.Lock()
565 wasCanceled := canceled
566 cmu.Unlock()
567 switch {
568 case err == nil:
569 case wasCanceled:
570 status = "canceled"
571 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
572 status, msg = "failed", "job_execution_timeout: "+runner.TimedOutLine(time.Duration(pl.Timeout)*time.Second, limitSource)
573 default:
574 status, msg = "failed", err.Error()
575 }
576 fctx, fc := context.WithTimeout(context.Background(), time.Minute)
577 defer fc()
578 if ferr := c.postJSON(fctx, base+"/finish", map[string]string{"status": status, "message": msg}, nil); ferr != nil {
579 c.log.Error("finish", "job", pl.JobID, "err", ferr)
580 }
581 c.jobs.Finish(pl.JobID, status)
582 c.log.Info("job done", "id", pl.JobID, "status", status)
583}
584
585func (c *client) upload(ctx context.Context, base, work string, paths []string) error {
586 pr, pw := io.Pipe()
587 go func() { pw.CloseWithError(runner.WriteTar(pw, work, paths)) }()
588 resp, err := c.do(ctx, http.MethodPut, base+"/artifacts", pr, "application/gzip")
589 if err != nil {
590 return fmt.Errorf("artifacts upload: %w", err)
591 }
592 defer resp.Body.Close()
593 if resp.StatusCode != http.StatusOK {
594 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
595 return fmt.Errorf("artifacts upload: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
596 }
597 return nil
598}