BlueGit

timfa bluegit

Files

bluegit / cmd / bluegit-runner / main.go

main.go

main.go 16.5 KiB Download Raw
1// Command bluegit-runner draait BlueGit CI-jobs op een eigen machine.
2// Hij maakt alleen uitgaande HTTPS-verbindingen (long-poll), er hoeft
3// geen poort open.
4package main
5
6import (
7 "bytes"
8 "context"
9 "encoding/json"
10 "errors"
11 "flag"
12 "fmt"
13 "io"
14 "log/slog"
15 "net/http"
16 "os"
17 "os/exec"
18 "os/signal"
19 "os/user"
20 "path/filepath"
21 "strconv"
22 "strings"
23 "sync"
24 "syscall"
25 "time"
26
27 "gopkg.in/yaml.v3"
28
29 "gitlab.com/Timfa/bluegit/internal/runner"
30 "gitlab.com/Timfa/bluegit/internal/runnercfg"
31)
32
33var version = "dev"
34
35const defaultConfig = "/etc/bluegit-runner/config.yml"
36
37// maxWorkers is de bovengrens voor concurrency.
38const maxWorkers = 64
39
40// Config staat in /etc/bluegit-runner/config.yml.
41type Config struct {
42 URL string `yaml:"url"`
43 Token string `yaml:"token"`
44 Name string `yaml:"name"`
45 Docker string `yaml:"docker"`
46 Git string `yaml:"git"`
47 WorkDir string `yaml:"work_dir"`
48 Network string `yaml:"network"`
49 Concurrency int `yaml:"concurrency"`
50 // UIListen is het adres van de webinterface; "off" zet hem uit.
51 UIListen string `yaml:"ui_listen"`
52 // Language is de standaardtaal van de webinterface (en, nl, de, fr).
53 Language string `yaml:"language"`
54}
55
56func (c *Config) defaults() {
57 if c.Docker == "" {
58 c.Docker = "docker"
59 }
60 if c.Git == "" {
61 c.Git = "git"
62 }
63 if c.WorkDir == "" {
64 c.WorkDir = "/var/lib/bluegit-runner"
65 }
66 if c.Network == "" {
67 c.Network = "bridge"
68 }
69 if c.Concurrency < 1 {
70 c.Concurrency = 1
71 }
72 if c.UIListen == "" {
73 c.UIListen = defaultUIListen
74 }
75 c.URL = strings.TrimRight(c.URL, "/")
76}
77
78func main() {
79 if len(os.Args) < 2 {
80 usage()
81 os.Exit(2)
82 }
83 var err error
84 switch os.Args[1] {
85 case "register":
86 err = cmdRegister(os.Args[2:])
87 case "run":
88 err = cmdRun(os.Args[2:])
89 case "ui-token":
90 err = cmdUIToken(os.Args[2:])
91 case "version", "--version":
92 fmt.Println(version)
93 default:
94 usage()
95 os.Exit(2)
96 }
97 if err != nil {
98 fmt.Fprintln(os.Stderr, "bluegit-runner:", err)
99 os.Exit(1)
100 }
101}
102
103func usage() {
104 fmt.Fprintln(os.Stderr, `usage:
105 bluegit-runner register --url https://git.example.com --token bgrr_... [--name NAME] [--config PATH]
106 bluegit-runner run [--config PATH]
107 bluegit-runner ui-token [--config PATH] (shows the web UI admin token)
108 bluegit-runner version`)
109}
110
111func loadConfig(path string) (Config, error) {
112 var c Config
113 b, err := os.ReadFile(path)
114 if err != nil {
115 return c, err
116 }
117 if err := yaml.Unmarshal(b, &c); err != nil {
118 return c, err
119 }
120 c.defaults()
121 if c.URL == "" || c.Token == "" {
122 return c, fmt.Errorf("%s: url and token are required; run bluegit-runner register first", path)
123 }
124 return c, nil
125}
126
127func cmdRegister(args []string) error {
128 fs := flag.NewFlagSet("register", flag.ExitOnError)
129 url := fs.String("url", "", "BlueGit base URL")
130 token := fs.String("token", "", "registration token from the BlueGit UI")
131 name := fs.String("name", "", "runner name (default: hostname)")
132 cfgPath := fs.String("config", defaultConfig, "config file")
133 _ = fs.Parse(args)
134 // Niet-interactief (agents): alles mag ook via de omgeving.
135 if *token == "" {
136 *token = os.Getenv("BLUEGIT_RUNNER_REGISTRATION_TOKEN")
137 }
138 if *url == "" {
139 *url = os.Getenv("BLUEGIT_RUNNER_URL")
140 }
141 if *name == "" {
142 *name = os.Getenv("BLUEGIT_RUNNER_NAME")
143 }
144 if *url == "" || *token == "" {
145 return errors.New("--url and --token are required")
146 }
147 if *name == "" {
148 *name, _ = os.Hostname()
149 }
150 var existing Config
151 if b, err := os.ReadFile(*cfgPath); err == nil {
152 _ = yaml.Unmarshal(b, &existing)
153 }
154 body, _ := json.Marshal(map[string]string{"token": *token, "name": *name, "version": version})
155 resp, err := http.Post(strings.TrimRight(*url, "/")+"/api/v1/runners/register", "application/json", bytes.NewReader(body))
156 if err != nil {
157 return err
158 }
159 defer resp.Body.Close()
160 var out struct {
161 ID int64 `json:"id"`
162 Name string `json:"name"`
163 Token string `json:"token"`
164 Message string `json:"message"`
165 }
166 _ = json.NewDecoder(resp.Body).Decode(&out)
167 if resp.StatusCode != http.StatusCreated || out.Token == "" {
168 return fmt.Errorf("registration failed (%d): %s", resp.StatusCode, out.Message)
169 }
170 existing.URL, existing.Token, existing.Name = strings.TrimRight(*url, "/"), out.Token, out.Name
171 b, err := yaml.Marshal(existing)
172 if err != nil {
173 return err
174 }
175 if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil {
176 return err
177 }
178 if err := os.WriteFile(*cfgPath, b, 0o600); err != nil {
179 return err
180 }
181 if os.Geteuid() == 0 {
182 if u, err := user.Lookup("bluegit-runner"); err == nil {
183 uid, _ := strconv.Atoi(u.Uid)
184 gid, _ := strconv.Atoi(u.Gid)
185 _ = os.Chown(*cfgPath, uid, gid)
186 }
187 }
188 fmt.Printf("registered runner %q (id %d); config written to %s\n", out.Name, out.ID, *cfgPath)
189 fmt.Printf("web UI: http://%s (admin token: sudo bluegit-runner ui-token)\n", defaultUIListen)
190 // Met het AUR-pakket: de service is al enabled en wacht op de config.
191 if os.Geteuid() == 0 && *cfgPath == defaultConfig {
192 if _, err := os.Stat("/usr/lib/systemd/system/bluegit-runner.service"); err == nil {
193 if err := exec.Command("systemctl", "enable", "--now", "bluegit-runner.service").Run(); err == nil {
194 _ = exec.Command("systemctl", "restart", "bluegit-runner.service").Run()
195 fmt.Println("bluegit-runner.service started")
196 return nil
197 }
198 }
199 }
200 fmt.Println("start it with: sudo systemctl enable --now bluegit-runner")
201 return nil
202}
203
204type client struct {
205 cfg Config
206 http *http.Client
207 log *slog.Logger
208 // beheer (webinterface)
209 file *runnercfg.File
210 jobs *runnercfg.Tracker
211 stMu sync.Mutex
212 seen time.Time
213 srvName string
214 srvPause bool
215}
216
217func (c *client) settings() runnercfg.Settings {
218 s, err := c.file.Load(c.defaults())
219 if err != nil {
220 return c.defaults()
221 }
222 return s
223}
224
225func (c *client) defaults() runnercfg.Settings {
226 d := runnercfg.Defaults()
227 d.Concurrency = c.cfg.Concurrency
228 return d
229}
230
231func (c *client) status() runnercfg.Status {
232 c.stMu.Lock()
233 defer c.stMu.Unlock()
234 name := c.settings().Name
235 if name == "" {
236 name = c.srvName
237 }
238 if name == "" {
239 name = c.cfg.Name
240 }
241 return runnercfg.Status{Kind: "standalone", Name: name, Version: version, ServerURL: c.cfg.URL,
242 Online: !c.seen.IsZero() && time.Since(c.seen) < 90*time.Second, LastContact: c.seen, ServerPause: c.srvPause}
243}
244
245// heartbeat meldt pauze en (lokaal ingestelde) naam aan de server.
246func (c *client) heartbeat(ctx context.Context) error {
247 s := c.settings()
248 var out struct {
249 Name string `json:"name"`
250 Paused bool `json:"paused"`
251 }
252 err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]any{"draining": s.Paused, "name": s.Name}, &out)
253 if err == nil {
254 c.stMu.Lock()
255 c.seen, c.srvName, c.srvPause = time.Now(), out.Name, out.Paused
256 c.stMu.Unlock()
257 }
258 return err
259}
260
261func cmdUIToken(args []string) error {
262 fs := flag.NewFlagSet("ui-token", flag.ExitOnError)
263 cfgPath := fs.String("config", defaultConfig, "config file")
264 _ = fs.Parse(args)
265 b, err := os.ReadFile(*cfgPath)
266 if err != nil {
267 return err
268 }
269 var cfg Config
270 if err := yaml.Unmarshal(b, &cfg); err != nil {
271 return err
272 }
273 cfg.defaults()
274 tok, _, err := ensureUIToken(cfg)
275 if err != nil {
276 return fmt.Errorf("%w (run as root or as bluegit-runner)", err)
277 }
278 fmt.Printf("web UI: http://%s\nadmin token: %s\n", cfg.UIListen, tok)
279 return nil
280}
281
282func (c *client) do(ctx context.Context, method, path string, body io.Reader, ctype string) (*http.Response, error) {
283 req, err := http.NewRequestWithContext(ctx, method, c.cfg.URL+path, body)
284 if err != nil {
285 return nil, err
286 }
287 req.Header.Set("Authorization", "Bearer "+c.cfg.Token)
288 req.Header.Set("X-Runner-Version", version)
289 if ctype != "" {
290 req.Header.Set("Content-Type", ctype)
291 }
292 return c.http.Do(req)
293}
294
295func (c *client) postJSON(ctx context.Context, path string, in, out any) error {
296 b, _ := json.Marshal(in)
297 resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b), "application/json")
298 if err != nil {
299 return err
300 }
301 defer resp.Body.Close()
302 if resp.StatusCode >= 300 {
303 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
304 return fmt.Errorf("%s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(msg)))
305 }
306 if out != nil {
307 return json.NewDecoder(resp.Body).Decode(out)
308 }
309 return nil
310}
311
312func cmdRun(args []string) error {
313 fs := flag.NewFlagSet("run", flag.ExitOnError)
314 cfgPath := fs.String("config", defaultConfig, "config file")
315 once := fs.Bool("once", false, "run at most one job, then exit (for tests)")
316 _ = fs.Parse(args)
317 cfg, err := loadConfig(*cfgPath)
318 if err != nil {
319 return err
320 }
321 log := slog.New(slog.NewTextHandler(os.Stderr, nil))
322 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
323 defer stop()
324 c := &client{cfg: cfg, http: &http.Client{Timeout: 0}, log: log,
325 file: &runnercfg.File{Path: filepath.Join(cfg.WorkDir, "settings.json")}, jobs: &runnercfg.Tracker{}}
326 c.jobs.SetPaused(c.settings().Paused)
327 if err := c.heartbeat(ctx); err != nil {
328 return fmt.Errorf("cannot reach BlueGit: %w", err)
329 }
330 log.Info("bluegit-runner started", "url", cfg.URL, "name", c.status().Name, "version", version, "concurrency", c.settings().Concurrency)
331 if cfg.UIListen != "off" {
332 tok, created, err := ensureUIToken(cfg)
333 if err != nil {
334 log.Warn("web UI token", "err", err)
335 } else {
336 h := &runnercfg.Handler{File: c.file, Default: c.defaults(), Tracker: c.jobs, WorkDir: cfg.WorkDir,
337 Status: c.status, Apply: func(runnercfg.Settings) error {
338 hctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
339 defer cancel()
340 _ = c.heartbeat(hctx) // naam en pauze meteen naar de server
341 return nil
342 }}
343 if err := serveUI(ctx, newWebUI(cfg, tok, h), cfg.UIListen, log.Warn); err != nil {
344 log.Warn("web UI", "listen", cfg.UIListen, "err", err)
345 } else {
346 log.Info("web UI", "url", "http://"+cfg.UIListen, "first_start", created, "token", "see: bluegit-runner ui-token")
347 }
348 }
349 }
350 go func() {
351 t := time.NewTicker(30 * time.Second)
352 defer t.Stop()
353 for {
354 select {
355 case <-ctx.Done():
356 return
357 case <-t.C:
358 if err := c.heartbeat(ctx); err != nil && ctx.Err() == nil {
359 log.Warn("heartbeat", "err", err)
360 }
361 }
362 }
363 }()
364 var wg sync.WaitGroup
365 // Een vaste pool; worker i werkt alleen als i < concurrency en niet gepauzeerd.
366 for i := 0; i < maxWorkers; i++ {
367 wg.Add(1)
368 go func(i int) {
369 defer wg.Done()
370 for ctx.Err() == nil {
371 s := c.settings()
372 if s.Paused || i >= s.Concurrency {
373 select {
374 case <-ctx.Done():
375 case <-time.After(2 * time.Second):
376 }
377 continue
378 }
379 ran, err := c.poll(ctx)
380 if err != nil && ctx.Err() == nil {
381 log.Warn("poll", "err", err)
382 select {
383 case <-ctx.Done():
384 case <-time.After(5 * time.Second):
385 }
386 }
387 if ran && *once {
388 stop()
389 }
390 }
391 }(i)
392 }
393 wg.Wait()
394 return nil
395}
396
397func (c *client) poll(ctx context.Context) (bool, error) {
398 pctx, cancel := context.WithTimeout(ctx, 60*time.Second)
399 defer cancel()
400 resp, err := c.do(pctx, http.MethodPost, "/api/v1/runner/jobs/request", strings.NewReader("{}"), "application/json")
401 if err != nil {
402 return false, err
403 }
404 defer resp.Body.Close()
405 if resp.StatusCode == http.StatusNoContent {
406 return false, nil
407 }
408 if resp.StatusCode != http.StatusOK {
409 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
410 return false, fmt.Errorf("request: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
411 }
412 var pl runner.RemotePayload
413 if err := json.NewDecoder(resp.Body).Decode(&pl); err != nil {
414 return false, err
415 }
416 c.runJob(ctx, pl)
417 return true, nil
418}
419
420// logBuffer verzamelt logregels en stuurt ze elke seconde door.
421type logBuffer struct {
422 mu sync.Mutex
423 buf strings.Builder
424}
425
426func (l *logBuffer) add(s string) {
427 l.mu.Lock()
428 l.buf.WriteString(s)
429 l.mu.Unlock()
430}
431
432func (l *logBuffer) take() string {
433 l.mu.Lock()
434 defer l.mu.Unlock()
435 s := l.buf.String()
436 l.buf.Reset()
437 return s
438}
439
440func (c *client) runJob(parent context.Context, pl runner.RemotePayload) {
441 c.log.Info("job", "id", pl.JobID, "name", pl.Job.Name)
442 base := fmt.Sprintf("/api/v1/runner/jobs/%d", pl.JobID)
443 ctx, cancel := context.WithCancel(parent)
444 defer cancel()
445 runCtx := ctx
446 limitSource := runner.TimeoutFromServer
447 if pl.Job.Timeout > 0 {
448 limitSource = runner.TimeoutFromJob
449 }
450 if m := c.settings().JobTimeoutMin; m > 0 && (pl.Timeout <= 0 || int64(m*60) < pl.Timeout) {
451 pl.Timeout = int64(m * 60)
452 limitSource = runner.TimeoutFromRunner
453 }
454 if pl.Timeout > 0 {
455 var tc context.CancelFunc
456 runCtx, tc = context.WithTimeout(ctx, time.Duration(pl.Timeout)*time.Second)
457 defer tc()
458 }
459 var canceled bool
460 var cmu sync.Mutex
461 logs := &logBuffer{}
462 send := func() {
463 chunk := logs.take()
464 var out struct {
465 Cancel bool `json:"cancel"`
466 }
467 sctx, sc := context.WithTimeout(context.Background(), 30*time.Second)
468 defer sc()
469 err := func() error {
470 resp, err := c.do(sctx, http.MethodPost, base+"/log", strings.NewReader(chunk), "text/plain")
471 if err != nil {
472 return err
473 }
474 defer resp.Body.Close()
475 if resp.StatusCode != http.StatusOK {
476 return fmt.Errorf("log: %d", resp.StatusCode)
477 }
478 return json.NewDecoder(resp.Body).Decode(&out)
479 }()
480 if err != nil {
481 // Bij een fout gaat de chunk terug in de buffer voor de volgende poging.
482 if chunk != "" {
483 logs.mu.Lock()
484 rest := logs.buf.String()
485 logs.buf.Reset()
486 logs.buf.WriteString(chunk + rest)
487 logs.mu.Unlock()
488 }
489 return
490 }
491 if out.Cancel {
492 cmu.Lock()
493 canceled = true
494 cmu.Unlock()
495 cancel()
496 }
497 }
498 done := make(chan struct{})
499 flushed := make(chan struct{})
500 go func() {
501 defer close(flushed)
502 t := time.NewTicker(time.Second)
503 defer t.Stop()
504 for {
505 select {
506 case <-done:
507 return
508 case <-t.C:
509 send()
510 }
511 }
512 }()
513 st := c.settings()
514 r := &runner.Runner{GitBin: c.cfg.Git, Docker: c.cfg.Docker, Storage: c.cfg.WorkDir, Network: c.cfg.Network, Name: c.status().Name}
515 if over, oerr := st.Overrides(c.cfg.WorkDir, nil); oerr == nil {
516 r.Configure(st, over)
517 }
518 c.jobs.Start(runnercfg.JobInfo{ID: pl.JobID, Name: pl.Job.Name, Pipeline: pl.PipelineID, Image: pl.Job.Image,
519 Project: pl.Env["CI_PROJECT_PATH"], URL: pl.Env["CI_JOB_URL"]})
520 work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))
521 defer func() {
522 if st.CleanupWork {
523 _ = runner.ClearWorkDir(work)
524 }
525 runnercfg.Cleanup(context.Background(), c.cfg.Docker, c.settings(), c.jobs, filepath.Join(c.cfg.WorkDir, "cache"))
526 }()
527 clone := func(ctx context.Context, dir string) error {
528 cmd := exec.CommandContext(ctx, c.cfg.Git, "clone", "--quiet", pl.CloneURL, dir)
529 // Het runner-token via de omgeving, niet in de proceslijst.
530 cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_COUNT=1",
531 "GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=Authorization: Bearer "+c.cfg.Token)
532 out, err := cmd.CombinedOutput()
533 if err != nil {
534 return fmt.Errorf("%s", strings.ReplaceAll(strings.TrimSpace(string(out)), c.cfg.Token, "[MASKED]"))
535 }
536 return nil
537 }
538 inputs := func(ctx context.Context, dir string) error {
539 resp, err := c.do(ctx, http.MethodGet, base+"/inputs", nil, "")
540 if err != nil {
541 return err
542 }
543 defer resp.Body.Close()
544 if resp.StatusCode != http.StatusOK {
545 return fmt.Errorf("inputs: %d", resp.StatusCode)
546 }
547 return runner.ReadTar(resp.Body, dir, nil, 0)
548 }
549 err := r.RunRemote(runCtx, pl, work, logs.add, clone, inputs)
550 if err == nil && len(pl.Job.ArtifactPaths) > 0 {
551 err = c.upload(runCtx, base, work, pl.Job.ArtifactPaths)
552 }
553 close(done)
554 <-flushed
555 send()
556 status, msg := "success", ""
557 cmu.Lock()
558 wasCanceled := canceled
559 cmu.Unlock()
560 switch {
561 case err == nil:
562 case wasCanceled:
563 status = "canceled"
564 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
565 status, msg = "failed", "job_execution_timeout: "+runner.TimedOutLine(time.Duration(pl.Timeout)*time.Second, limitSource)
566 default:
567 status, msg = "failed", err.Error()
568 }
569 fctx, fc := context.WithTimeout(context.Background(), time.Minute)
570 defer fc()
571 if ferr := c.postJSON(fctx, base+"/finish", map[string]string{"status": status, "message": msg}, nil); ferr != nil {
572 c.log.Error("finish", "job", pl.JobID, "err", ferr)
573 }
574 c.jobs.Finish(pl.JobID, status)
575 c.log.Info("job done", "id", pl.JobID, "status", status)
576}
577
578func (c *client) upload(ctx context.Context, base, work string, paths []string) error {
579 pr, pw := io.Pipe()
580 go func() { pw.CloseWithError(runner.WriteTar(pw, work, paths)) }()
581 resp, err := c.do(ctx, http.MethodPut, base+"/artifacts", pr, "application/gzip")
582 if err != nil {
583 return fmt.Errorf("artifacts upload: %w", err)
584 }
585 defer resp.Body.Close()
586 if resp.StatusCode != http.StatusOK {
587 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
588 return fmt.Errorf("artifacts upload: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
589 }
590 return nil
591}