BlueGit

timfa bluegit

Files

bluegit / cmd / bluegit-runner / main.go

main.go

main.go 17.0 KiB Download Raw
1// Command bluegit-runner draait BlueGit CI-jobs op een eigen machine.
2// Hij maakt alleen uitgaande HTTPS-verbindingen (long-poll), er hoeft
3// geen poort open.
4package main
5
6import (
7 "bytes"
8 "context"
9 "encoding/json"
10 "errors"
11 "flag"
12 "fmt"
13 "io"
14 "log/slog"
15 "net/http"
16 "os"
17 "os/exec"
18 "os/signal"
19 "os/user"
20 "path/filepath"
21 "strconv"
22 "strings"
23 "sync"
24 "syscall"
25 "time"
26
27 "gopkg.in/yaml.v3"
28
29 "gitlab.com/Timfa/bluegit/internal/apppaths"
30 "gitlab.com/Timfa/bluegit/internal/runner"
31 "gitlab.com/Timfa/bluegit/internal/runnercfg"
32)
33
34var version = "dev"
35
36var defaultConfig = apppaths.ConfigFile("", "BLUEGIT_RUNNER_CONFIG", "", "/etc/bluegit-runner/config.yml", "BlueGit-runner")
37
38// maxWorkers is de bovengrens voor concurrency.
39const maxWorkers = 64
40
41// Config staat in /etc/bluegit-runner/config.yml, of op Windows in
42// %LOCALAPPDATA%\BlueGit-runner\config.yml.
43type Config struct {
44 URL string `yaml:"url"`
45 Token string `yaml:"token"`
46 Name string `yaml:"name"`
47 Docker string `yaml:"docker"`
48 Git string `yaml:"git"`
49 WorkDir string `yaml:"work_dir"`
50 Network string `yaml:"network"`
51 Concurrency int `yaml:"concurrency"`
52 // UIListen is het adres van de webinterface; "off" zet hem uit.
53 UIListen string `yaml:"ui_listen"`
54 // Language is de standaardtaal van de webinterface (en, nl, de, fr).
55 Language string `yaml:"language"`
56}
57
58func (c *Config) defaults() {
59 if c.Docker == "" {
60 c.Docker = "docker"
61 }
62 if c.Git == "" {
63 c.Git = "git"
64 }
65 if c.WorkDir == "" {
66 c.WorkDir = apppaths.DataDir("/var/lib/bluegit-runner", "BlueGit-runner")
67 }
68 if c.Network == "" {
69 c.Network = "bridge"
70 }
71 if c.Concurrency < 1 {
72 c.Concurrency = 1
73 }
74 if c.UIListen == "" {
75 c.UIListen = defaultUIListen
76 }
77 c.URL = strings.TrimRight(c.URL, "/")
78}
79
80func main() {
81 if len(os.Args) < 2 {
82 usage()
83 os.Exit(2)
84 }
85 var err error
86 switch os.Args[1] {
87 case "register":
88 err = cmdRegister(os.Args[2:])
89 case "run":
90 err = cmdRun(os.Args[2:])
91 case "ui-token":
92 err = cmdUIToken(os.Args[2:])
93 case "version", "--version":
94 fmt.Println(version)
95 default:
96 usage()
97 os.Exit(2)
98 }
99 if err != nil {
100 fmt.Fprintln(os.Stderr, "bluegit-runner:", err)
101 os.Exit(1)
102 }
103}
104
105func usage() {
106 fmt.Fprintln(os.Stderr, `usage:
107 bluegit-runner register --url https://git.example.com --token bgrr_... [--name NAME] [--config PATH]
108 bluegit-runner run [--config PATH]
109 bluegit-runner ui-token [--config PATH] (shows the web UI admin token)
110 bluegit-runner version`)
111}
112
113func loadConfig(path string) (Config, error) {
114 var c Config
115 b, err := os.ReadFile(path)
116 if err != nil {
117 return c, err
118 }
119 if err := yaml.Unmarshal(b, &c); err != nil {
120 return c, err
121 }
122 c.defaults()
123 if c.URL == "" || c.Token == "" {
124 return c, fmt.Errorf("%s: url and token are required; run bluegit-runner register first", path)
125 }
126 return c, nil
127}
128
129func cmdRegister(args []string) error {
130 fs := flag.NewFlagSet("register", flag.ExitOnError)
131 url := fs.String("url", "", "BlueGit base URL")
132 token := fs.String("token", "", "registration token from the BlueGit UI")
133 name := fs.String("name", "", "runner name (default: hostname)")
134 cfgPath := fs.String("config", defaultConfig, "config file")
135 _ = fs.Parse(args)
136 // Niet-interactief (agents): alles mag ook via de omgeving.
137 if *token == "" {
138 *token = os.Getenv("BLUEGIT_RUNNER_REGISTRATION_TOKEN")
139 }
140 if *url == "" {
141 *url = os.Getenv("BLUEGIT_RUNNER_URL")
142 }
143 if *name == "" {
144 *name = os.Getenv("BLUEGIT_RUNNER_NAME")
145 }
146 if *url == "" || *token == "" {
147 return errors.New("--url and --token are required")
148 }
149 if *name == "" {
150 *name, _ = os.Hostname()
151 }
152 var existing Config
153 if b, err := os.ReadFile(*cfgPath); err == nil {
154 _ = yaml.Unmarshal(b, &existing)
155 }
156 body, _ := json.Marshal(map[string]string{"token": *token, "name": *name, "version": version})
157 resp, err := http.Post(strings.TrimRight(*url, "/")+"/api/v1/runners/register", "application/json", bytes.NewReader(body))
158 if err != nil {
159 return err
160 }
161 defer resp.Body.Close()
162 var out struct {
163 ID int64 `json:"id"`
164 Name string `json:"name"`
165 Token string `json:"token"`
166 Message string `json:"message"`
167 }
168 _ = json.NewDecoder(resp.Body).Decode(&out)
169 if resp.StatusCode != http.StatusCreated || out.Token == "" {
170 return fmt.Errorf("registration failed (%d): %s", resp.StatusCode, out.Message)
171 }
172 existing.URL, existing.Token, existing.Name = strings.TrimRight(*url, "/"), out.Token, out.Name
173 b, err := yaml.Marshal(existing)
174 if err != nil {
175 return err
176 }
177 if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil {
178 return err
179 }
180 if err := os.WriteFile(*cfgPath, b, 0o600); err != nil {
181 return err
182 }
183 if os.Geteuid() == 0 {
184 if u, err := user.Lookup("bluegit-runner"); err == nil {
185 uid, _ := strconv.Atoi(u.Uid)
186 gid, _ := strconv.Atoi(u.Gid)
187 _ = os.Chown(*cfgPath, uid, gid)
188 }
189 }
190 fmt.Printf("registered runner %q (id %d); config written to %s\n", out.Name, out.ID, *cfgPath)
191 fmt.Printf("web UI: http://%s (admin token: sudo bluegit-runner ui-token)\n", defaultUIListen)
192 // Met het AUR-pakket: de service is al enabled en wacht op de config.
193 if os.Geteuid() == 0 && *cfgPath == defaultConfig {
194 if _, err := os.Stat("/usr/lib/systemd/system/bluegit-runner.service"); err == nil {
195 if err := exec.Command("systemctl", "enable", "--now", "bluegit-runner.service").Run(); err == nil {
196 _ = exec.Command("systemctl", "restart", "bluegit-runner.service").Run()
197 fmt.Println("bluegit-runner.service started")
198 return nil
199 }
200 }
201 }
202 fmt.Println("start it with: sudo systemctl enable --now bluegit-runner")
203 return nil
204}
205
206type client struct {
207 cfg Config
208 http *http.Client
209 log *slog.Logger
210 // beheer (webinterface)
211 file *runnercfg.File
212 jobs *runnercfg.Tracker
213 stMu sync.Mutex
214 seen time.Time
215 srvName string
216 srvPause bool
217}
218
219func (c *client) settings() runnercfg.Settings {
220 s, err := c.file.Load(c.defaults())
221 if err != nil {
222 return c.defaults()
223 }
224 return s
225}
226
227func (c *client) defaults() runnercfg.Settings {
228 d := runnercfg.Defaults()
229 d.Concurrency = c.cfg.Concurrency
230 return d
231}
232
233func (c *client) status() runnercfg.Status {
234 c.stMu.Lock()
235 defer c.stMu.Unlock()
236 name := c.settings().Name
237 if name == "" {
238 name = c.srvName
239 }
240 if name == "" {
241 name = c.cfg.Name
242 }
243 docker := runnercfg.PingDocker(context.Background(), c.cfg.Docker)
244 online := !c.seen.IsZero() && time.Since(c.seen) < 90*time.Second && docker.Reachable
245 return runnercfg.Status{Kind: "standalone", Name: name, Version: version, ServerURL: c.cfg.URL,
246 Online: online, LastContact: c.seen, ServerPause: c.srvPause, Docker: docker}
247}
248
249// heartbeat meldt pauze en (lokaal ingestelde) naam aan de server.
250func (c *client) heartbeat(ctx context.Context) error {
251 s := c.settings()
252 var out struct {
253 Name string `json:"name"`
254 Paused bool `json:"paused"`
255 }
256 err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]any{"draining": s.Paused, "name": s.Name}, &out)
257 if err == nil {
258 c.stMu.Lock()
259 c.seen, c.srvName, c.srvPause = time.Now(), out.Name, out.Paused
260 c.stMu.Unlock()
261 }
262 return err
263}
264
265func cmdUIToken(args []string) error {
266 fs := flag.NewFlagSet("ui-token", flag.ExitOnError)
267 cfgPath := fs.String("config", defaultConfig, "config file")
268 _ = fs.Parse(args)
269 b, err := os.ReadFile(*cfgPath)
270 if err != nil {
271 return err
272 }
273 var cfg Config
274 if err := yaml.Unmarshal(b, &cfg); err != nil {
275 return err
276 }
277 cfg.defaults()
278 tok, _, err := ensureUIToken(cfg)
279 if err != nil {
280 return fmt.Errorf("%w (run as root or as bluegit-runner)", err)
281 }
282 fmt.Printf("web UI: http://%s\nadmin token: %s\n", cfg.UIListen, tok)
283 return nil
284}
285
286func (c *client) do(ctx context.Context, method, path string, body io.Reader, ctype string) (*http.Response, error) {
287 req, err := http.NewRequestWithContext(ctx, method, c.cfg.URL+path, body)
288 if err != nil {
289 return nil, err
290 }
291 req.Header.Set("Authorization", "Bearer "+c.cfg.Token)
292 req.Header.Set("X-Runner-Version", version)
293 if ctype != "" {
294 req.Header.Set("Content-Type", ctype)
295 }
296 return c.http.Do(req)
297}
298
299func (c *client) postJSON(ctx context.Context, path string, in, out any) error {
300 b, _ := json.Marshal(in)
301 resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b), "application/json")
302 if err != nil {
303 return err
304 }
305 defer resp.Body.Close()
306 if resp.StatusCode >= 300 {
307 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
308 return fmt.Errorf("%s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(msg)))
309 }
310 if out != nil {
311 return json.NewDecoder(resp.Body).Decode(out)
312 }
313 return nil
314}
315
316func cmdRun(args []string) error {
317 fs := flag.NewFlagSet("run", flag.ExitOnError)
318 cfgPath := fs.String("config", defaultConfig, "config file")
319 once := fs.Bool("once", false, "run at most one job, then exit (for tests)")
320 _ = fs.Parse(args)
321 cfg, err := loadConfig(*cfgPath)
322 if err != nil {
323 return err
324 }
325 log := slog.New(slog.NewTextHandler(os.Stderr, nil))
326 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
327 defer stop()
328 c := &client{cfg: cfg, http: &http.Client{Timeout: 0}, log: log,
329 file: &runnercfg.File{Path: filepath.Join(cfg.WorkDir, "settings.json")}, jobs: &runnercfg.Tracker{}}
330 c.jobs.SetPaused(c.settings().Paused)
331 if err := c.heartbeat(ctx); err != nil {
332 return fmt.Errorf("cannot reach BlueGit: %w", err)
333 }
334 log.Info("bluegit-runner started", "url", cfg.URL, "name", c.status().Name, "version", version, "concurrency", c.settings().Concurrency)
335 if cfg.UIListen != "off" {
336 tok, created, err := ensureUIToken(cfg)
337 if err != nil {
338 log.Warn("web UI token", "err", err)
339 } else {
340 h := &runnercfg.Handler{File: c.file, Default: c.defaults(), Tracker: c.jobs, WorkDir: cfg.WorkDir,
341 Status: c.status, Apply: func(runnercfg.Settings) error {
342 hctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
343 defer cancel()
344 _ = c.heartbeat(hctx) // naam en pauze meteen naar de server
345 return nil
346 }}
347 if err := serveUI(ctx, newWebUI(cfg, tok, h), cfg.UIListen, log.Warn); err != nil {
348 log.Warn("web UI", "listen", cfg.UIListen, "err", err)
349 } else {
350 log.Info("web UI", "url", "http://"+cfg.UIListen, "first_start", created, "token", "see: bluegit-runner ui-token")
351 }
352 }
353 }
354 go func() {
355 t := time.NewTicker(30 * time.Second)
356 defer t.Stop()
357 for {
358 select {
359 case <-ctx.Done():
360 return
361 case <-t.C:
362 if err := c.heartbeat(ctx); err != nil && ctx.Err() == nil {
363 log.Warn("heartbeat", "err", err)
364 }
365 }
366 }
367 }()
368 var wg sync.WaitGroup
369 // Een vaste pool; worker i werkt alleen als i < concurrency en niet gepauzeerd.
370 for i := 0; i < maxWorkers; i++ {
371 wg.Add(1)
372 go func(i int) {
373 defer wg.Done()
374 for ctx.Err() == nil {
375 s := c.settings()
376 if s.Paused || i >= s.Concurrency {
377 select {
378 case <-ctx.Done():
379 case <-time.After(2 * time.Second):
380 }
381 continue
382 }
383 ran, err := c.poll(ctx)
384 if err != nil && ctx.Err() == nil {
385 log.Warn("poll", "err", err)
386 select {
387 case <-ctx.Done():
388 case <-time.After(5 * time.Second):
389 }
390 }
391 if ran && *once {
392 stop()
393 }
394 }
395 }(i)
396 }
397 wg.Wait()
398 return nil
399}
400
401func (c *client) poll(ctx context.Context) (bool, error) {
402 pctx, cancel := context.WithTimeout(ctx, 60*time.Second)
403 defer cancel()
404 resp, err := c.do(pctx, http.MethodPost, "/api/v1/runner/jobs/request", strings.NewReader("{}"), "application/json")
405 if err != nil {
406 return false, err
407 }
408 defer resp.Body.Close()
409 if resp.StatusCode == http.StatusNoContent {
410 return false, nil
411 }
412 if resp.StatusCode != http.StatusOK {
413 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
414 return false, fmt.Errorf("request: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
415 }
416 var pl runner.RemotePayload
417 if err := json.NewDecoder(resp.Body).Decode(&pl); err != nil {
418 return false, err
419 }
420 c.runJob(ctx, pl)
421 return true, nil
422}
423
424// logBuffer verzamelt logregels en stuurt ze elke seconde door.
425type logBuffer struct {
426 mu sync.Mutex
427 buf strings.Builder
428}
429
430func (l *logBuffer) add(s string) {
431 l.mu.Lock()
432 l.buf.WriteString(s)
433 l.mu.Unlock()
434}
435
436func (l *logBuffer) take() string {
437 l.mu.Lock()
438 defer l.mu.Unlock()
439 s := l.buf.String()
440 l.buf.Reset()
441 return s
442}
443
444func (c *client) runJob(parent context.Context, pl runner.RemotePayload) {
445 c.log.Info("job", "id", pl.JobID, "name", pl.Job.Name)
446 base := fmt.Sprintf("/api/v1/runner/jobs/%d", pl.JobID)
447 if pl.Job.DoNotInterrupt {
448 // Stoppen van de runner (SIGTERM) breekt deze job niet af; de
449 // workers wachten tot hij klaar is. Annuleren met force werkt wel.
450 parent = context.WithoutCancel(parent)
451 }
452 ctx, cancel := context.WithCancel(parent)
453 defer cancel()
454 runCtx := ctx
455 limitSource := runner.TimeoutFromServer
456 if pl.Job.Timeout > 0 {
457 limitSource = runner.TimeoutFromJob
458 }
459 if m := c.settings().JobTimeoutMin; m > 0 && (pl.Timeout <= 0 || int64(m*60) < pl.Timeout) {
460 pl.Timeout = int64(m * 60)
461 limitSource = runner.TimeoutFromRunner
462 }
463 if pl.Timeout > 0 {
464 var tc context.CancelFunc
465 runCtx, tc = context.WithTimeout(ctx, time.Duration(pl.Timeout)*time.Second)
466 defer tc()
467 }
468 var canceled bool
469 var cmu sync.Mutex
470 logs := &logBuffer{}
471 send := func() {
472 chunk := logs.take()
473 var out struct {
474 Cancel bool `json:"cancel"`
475 }
476 sctx, sc := context.WithTimeout(context.Background(), 30*time.Second)
477 defer sc()
478 err := func() error {
479 resp, err := c.do(sctx, http.MethodPost, base+"/log", strings.NewReader(chunk), "text/plain")
480 if err != nil {
481 return err
482 }
483 defer resp.Body.Close()
484 if resp.StatusCode != http.StatusOK {
485 return fmt.Errorf("log: %d", resp.StatusCode)
486 }
487 return json.NewDecoder(resp.Body).Decode(&out)
488 }()
489 if err != nil {
490 // Bij een fout gaat de chunk terug in de buffer voor de volgende poging.
491 if chunk != "" {
492 logs.mu.Lock()
493 rest := logs.buf.String()
494 logs.buf.Reset()
495 logs.buf.WriteString(chunk + rest)
496 logs.mu.Unlock()
497 }
498 return
499 }
500 if out.Cancel {
501 cmu.Lock()
502 canceled = true
503 cmu.Unlock()
504 cancel()
505 }
506 }
507 done := make(chan struct{})
508 flushed := make(chan struct{})
509 go func() {
510 defer close(flushed)
511 t := time.NewTicker(time.Second)
512 defer t.Stop()
513 for {
514 select {
515 case <-done:
516 return
517 case <-t.C:
518 send()
519 }
520 }
521 }()
522 st := c.settings()
523 r := &runner.Runner{GitBin: c.cfg.Git, Docker: c.cfg.Docker, Storage: c.cfg.WorkDir, Network: c.cfg.Network, Name: c.status().Name}
524 if over, oerr := st.Overrides(c.cfg.WorkDir, nil); oerr == nil {
525 r.Configure(st, over)
526 }
527 c.jobs.Start(runnercfg.JobInfo{ID: pl.JobID, Name: pl.Job.Name, Pipeline: pl.PipelineID, Image: pl.Job.Image,
528 Project: pl.Env["CI_PROJECT_PATH"], URL: pl.Env["CI_JOB_URL"]})
529 work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))
530 defer func() {
531 if st.CleanupWork {
532 _ = runner.ClearWorkDir(work)
533 }
534 runnercfg.Cleanup(context.Background(), c.cfg.Docker, c.settings(), c.jobs, filepath.Join(c.cfg.WorkDir, "cache"))
535 }()
536 clone := func(ctx context.Context, dir string) error {
537 cmd := exec.CommandContext(ctx, c.cfg.Git, "clone", "--quiet", pl.CloneURL, dir)
538 // Het runner-token via de omgeving, niet in de proceslijst.
539 cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_COUNT=1",
540 "GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=Authorization: Bearer "+c.cfg.Token)
541 out, err := cmd.CombinedOutput()
542 if err != nil {
543 return fmt.Errorf("%s", strings.ReplaceAll(strings.TrimSpace(string(out)), c.cfg.Token, "[MASKED]"))
544 }
545 return nil
546 }
547 inputs := func(ctx context.Context, dir string) error {
548 resp, err := c.do(ctx, http.MethodGet, base+"/inputs", nil, "")
549 if err != nil {
550 return err
551 }
552 defer resp.Body.Close()
553 if resp.StatusCode != http.StatusOK {
554 return fmt.Errorf("inputs: %d", resp.StatusCode)
555 }
556 return runner.ReadTar(resp.Body, dir, nil, 0)
557 }
558 err := r.RunRemote(runCtx, pl, work, logs.add, clone, inputs)
559 if err == nil && len(pl.Job.ArtifactPaths) > 0 {
560 err = c.upload(runCtx, base, work, pl.Job.ArtifactPaths)
561 }
562 close(done)
563 <-flushed
564 send()
565 status, msg := "success", ""
566 cmu.Lock()
567 wasCanceled := canceled
568 cmu.Unlock()
569 switch {
570 case err == nil:
571 case wasCanceled:
572 status = "canceled"
573 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
574 status, msg = "failed", "job_execution_timeout: "+runner.TimedOutLine(time.Duration(pl.Timeout)*time.Second, limitSource)
575 default:
576 status, msg = "failed", err.Error()
577 }
578 fctx, fc := context.WithTimeout(context.Background(), time.Minute)
579 defer fc()
580 if ferr := c.postJSON(fctx, base+"/finish", map[string]string{"status": status, "message": msg}, nil); ferr != nil {
581 c.log.Error("finish", "job", pl.JobID, "err", ferr)
582 }
583 c.jobs.Finish(pl.JobID, status)
584 c.log.Info("job done", "id", pl.JobID, "status", status)
585}
586
587func (c *client) upload(ctx context.Context, base, work string, paths []string) error {
588 pr, pw := io.Pipe()
589 go func() { pw.CloseWithError(runner.WriteTar(pw, work, paths)) }()
590 resp, err := c.do(ctx, http.MethodPut, base+"/artifacts", pr, "application/gzip")
591 if err != nil {
592 return fmt.Errorf("artifacts upload: %w", err)
593 }
594 defer resp.Body.Close()
595 if resp.StatusCode != http.StatusOK {
596 msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
597 return fmt.Errorf("artifacts upload: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
598 }
599 return nil
600}