BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

feat: GitLab-achtige blue-ci-cd.yml
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
Failed 17caa14a History
runner.go 15.1 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "fmt"
8 "io"
9 "log/slog"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "strings"
14 "sync"
15 "time"
16
17 "gitlab.com/Timfa/bluegit/internal/access"
18 "gitlab.com/Timfa/bluegit/internal/ci"
19 "gitlab.com/Timfa/bluegit/internal/gitrepo"
20 "gitlab.com/Timfa/bluegit/internal/store"
21 "gitlab.com/Timfa/bluegit/internal/webhook"
22)
23
24// Runner plant jobs en stuurt het resultaat door naar webhooks.
25type Runner struct {
26 Store *store.Store
27 GitBin string
28 Docker string
29 Storage string
30 BaseURL string
31 Network string
32 Timeout time.Duration
33 Log *slog.Logger
34 OnEvent func(repoID int64, ev webhook.Event)
35 slots chan struct{}
36 mu sync.RWMutex
37}
38
39// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
40func (r *Runner) SetStorage(path string) {
41 r.mu.Lock()
42 r.Storage = path
43 r.mu.Unlock()
44}
45
46func (r *Runner) storage() string {
47 r.mu.RLock()
48 defer r.mu.RUnlock()
49 return r.Storage
50}
51
52// Loop blijft jobs oppakken tot ctx afloopt.
53func (r *Runner) Loop(ctx context.Context, concurrency int) {
54 if concurrency < 1 {
55 concurrency = 1
56 }
57 r.slots = make(chan struct{}, concurrency)
58 tick := time.NewTicker(2 * time.Second)
59 defer tick.Stop()
60 for {
61 select {
62 case <-ctx.Done():
63 return
64 case <-tick.C:
65 r.Tick(ctx)
66 }
67 }
68}
69
70// Tick bekijkt alle open pipelines één keer.
71// Zonder opslagpad gebeurt er niets.
72func (r *Runner) Tick(ctx context.Context) {
73 if r.storage() == "" {
74 return
75 }
76 pipes, err := r.Store.ActivePipelines(ctx)
77 if err != nil {
78 r.Log.Error("pipelines", "err", err)
79 return
80 }
81 for _, p := range pipes {
82 r.schedule(ctx, p)
83 }
84}
85
86func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
87 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
88 if err != nil {
89 r.Log.Error("jobs", "err", err)
90 return
91 }
92 specs := make([]ci.Job, len(jobs))
93 status := map[string]string{}
94 for i, j := range jobs {
95 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
96 status[j.Name] = j.Status
97 }
98 for _, j := range jobs {
99 if j.Status != "queued" && j.Status != "manual" {
100 continue
101 }
102 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
103 case "skip":
104 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
105 status[j.Name] = "skipped"
106 case "fail":
107 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
108 status[j.Name] = "failed"
109 case "ready":
110 if j.Status != "queued" {
111 continue
112 }
113 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
114 continue
115 }
116 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
117 if err != nil || !ok {
118 continue
119 }
120 status[j.Name] = "running"
121 job := j
122 pipe := p
123 go r.execute(ctx, pipe, job)
124 }
125 }
126}
127
128func indexOf(jobs []store.Job, name string) int {
129 for i, j := range jobs {
130 if j.Name == name {
131 return i
132 }
133 }
134 return 0
135}
136
137func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
138 if r.slots != nil {
139 r.slots <- struct{}{}
140 defer func() { <-r.slots }()
141 }
142 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
143 mask, err := r.runOne(ctx, p, job)
144 status := "success"
145 extra := ""
146 if err != nil {
147 status = "failed"
148 extra = stamp(ci.Mask(err.Error(), mask))
149 }
150 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
151 r.Log.Error("job afronden", "err", ferr)
152 }
153 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
154 if rerr != nil {
155 r.Log.Error("pipelinestatus", "err", rerr)
156 return
157 }
158 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
159 kind := webhook.EventPipelineSuccess
160 word := "geslaagd"
161 if final == "failed" {
162 kind = webhook.EventPipelineFailure
163 word = "mislukt"
164 }
165 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
166 if err != nil {
167 return
168 }
169 branch, _ := gitrepo.BranchOfRef(p.Ref)
170 if branch == "" {
171 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
172 }
173 r.OnEvent(p.RepoID, webhook.Event{
174 Type: kind,
175 Project: repo.FullPath(),
176 Branch: branch,
177 Commit: p.CommitSHA,
178 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
179 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
180 PipelineID: fmt.Sprint(p.ID),
181 })
182 }
183}
184
185func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
186 limit := r.Timeout
187 if job.Timeout > 0 {
188 limit = job.Timeout
189 }
190 if limit > 0 {
191 var cancel context.CancelFunc
192 ctx, cancel = context.WithTimeout(ctx, limit)
193 defer cancel()
194 }
195 repo, err := r.Store.RepoByID(ctx, p.RepoID)
196 if err != nil {
197 return nil, err
198 }
199 sink := &logSink{store: r.Store, jobID: job.ID}
200 sink.section("prepare", "Preparing the docker executor")
201 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), job.Name)
202 _ = os.RemoveAll(work)
203 if err := os.MkdirAll(work, 0o755); err != nil {
204 sink.closeSection()
205 return nil, err
206 }
207 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
208 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
209 sink.closeSection()
210 return nil, fmt.Errorf("clone: %w", err)
211 }
212 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
213 sink.closeSection()
214 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
215 }
216 if err := r.restoreCache(repo.ID, job, work); err != nil {
217 sink.closeSection()
218 return nil, err
219 }
220 if err := r.materialize(ctx, p, job, work); err != nil {
221 sink.closeSection()
222 return nil, err
223 }
224 scriptPath := filepath.Join(work, ".bluegit-job.sh")
225 if err := os.WriteFile(scriptPath, []byte(shellScript(job)), 0o755); err != nil {
226 sink.closeSection()
227 return nil, err
228 }
229 token, err := r.Store.CreateJobToken(ctx, job.ID)
230 if err != nil {
231 sink.closeSection()
232 return nil, err
233 }
234 env, mask, err := r.environment(ctx, repo, p, job, token)
235 if err != nil {
236 sink.closeSection()
237 return nil, err
238 }
239 sink.secrets = mask
240 if _, err := exec.LookPath(r.Docker); err != nil {
241 sink.closeSection()
242 return mask, fmt.Errorf("docker (%s) is not available; the job only runs in a container", r.Docker)
243 }
244 sink.closeSection()
245 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
246 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
247 // staat de waarde niet in de proceslijst.
248 args := []string{"run", "--rm", "--network", r.Network, "-v", work + ":/workspace", "-w", "/workspace"}
249 cmdEnv := append([]string{}, os.Environ()...)
250 for k, v := range env {
251 args = append(args, "-e", k)
252 cmdEnv = append(cmdEnv, k+"="+v)
253 }
254 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
255 cmd := exec.CommandContext(ctx, r.Docker, args...)
256 cmd.Env = cmdEnv
257 cmd.Stdout = sink
258 cmd.Stderr = sink
259 err = cmd.Run()
260 sink.flush()
261 if err != nil {
262 return mask, fmt.Errorf("container ended with an error: %w", err)
263 }
264 if err := r.collect(ctx, job, work); err != nil {
265 return mask, err
266 }
267 if err := r.saveCache(repo.ID, job, work); err != nil {
268 return mask, err
269 }
270 return mask, nil
271}
272
273func shellScript(job store.Job) string {
274 var b strings.Builder
275 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
276 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
277 if strings.TrimSpace(job.Before) != "" {
278 b.WriteString("(\nset -e\n")
279 b.WriteString(job.Before)
280 if !strings.HasSuffix(job.Before, "\n") {
281 b.WriteByte('\n')
282 }
283 b.WriteString(")\nbrc=$?\n")
284 } else {
285 b.WriteString("brc=0\n")
286 }
287 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
288 b.WriteString("echo \"section_start:script:Running script\"\n")
289 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
290 b.WriteString(job.Script)
291 if !strings.HasSuffix(job.Script, "\n") {
292 b.WriteByte('\n')
293 }
294 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
295 b.WriteString("echo \"section_end:script:Running script\"\n")
296 b.WriteString("echo \"section_start:after:After script\"\n")
297 if strings.TrimSpace(job.After) != "" {
298 b.WriteString("(\nset -e\n")
299 b.WriteString(job.After)
300 if !strings.HasSuffix(job.After, "\n") {
301 b.WriteByte('\n')
302 }
303 b.WriteString(")\n")
304 }
305 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
306 return b.String()
307}
308
309func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
310 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
311 if err != nil {
312 return err
313 }
314 wanted := map[string]bool{}
315 if job.NeedsSet {
316 for _, n := range job.Needs {
317 wanted[n] = true
318 }
319 } else {
320 stageOrder := map[string]int{}
321 n := 0
322 for _, j := range jobs {
323 if _, ok := stageOrder[j.Stage]; !ok {
324 stageOrder[j.Stage] = n
325 n++
326 }
327 }
328 my := stageOrder[job.Stage]
329 for _, j := range jobs {
330 if stageOrder[j.Stage] < my {
331 wanted[j.Name] = true
332 }
333 }
334 }
335 for _, j := range jobs {
336 if !wanted[j.Name] || j.Status != "success" {
337 continue
338 }
339 for _, rel := range j.ArtifactPaths {
340 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
341 if err != nil {
342 continue
343 }
344 dst := filepath.Join(work, filepath.FromSlash(rel))
345 if err := copyFile(src, dst); err != nil {
346 return err
347 }
348 }
349 }
350 return nil
351}
352
353func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
354 var expires *time.Time
355 if job.ArtifactExpire > 0 {
356 t := time.Now().Add(job.ArtifactExpire)
357 expires = &t
358 }
359 for _, rel := range job.ArtifactPaths {
360 src := filepath.Join(work, filepath.FromSlash(rel))
361 info, err := os.Stat(src)
362 if err != nil || info.IsDir() {
363 return fmt.Errorf("artifact %s ontbreekt", rel)
364 }
365 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
366 if err := copyFile(src, dst); err != nil {
367 return err
368 }
369 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
370 return err
371 }
372 }
373 return nil
374}
375
376func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
377 name, _ := ci.SplitRef(p.Ref)
378 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
379 if err != nil {
380 return nil, nil, err
381 }
382 refProtected := access.BranchProtected(patterns, name)
383 env := map[string]string{}
384 var mask []string
385 if repo.OwnerKind == "group" {
386 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
387 if err != nil {
388 return nil, nil, err
389 }
390 for k, v := range groupSecrets {
391 env[k] = v
392 mask = append(mask, v)
393 }
394 }
395 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
396 if err != nil {
397 return nil, nil, err
398 }
399 for k, v := range repoSecrets {
400 env[k] = v
401 mask = append(mask, v)
402 }
403 for k, v := range job.Variables {
404 env[k] = v
405 }
406 base := strings.TrimRight(r.BaseURL, "/")
407 pre := ci.Variables(ci.Ref{
408 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle,
409 DefaultBranch: repo.DefaultBranch, Source: p.Source,
410 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
411 APIURL: base + "/api/v1", PipelineID: p.ID,
412 })
413 for k, v := range pre {
414 env[k] = v
415 }
416 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
417 env["CI_JOB_NAME"] = job.Name
418 env["CI_JOB_STAGE"] = job.Stage
419 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
420 env["CI_JOB_TOKEN"] = token
421 mask = append(mask, token)
422 return env, mask, nil
423}
424
425func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
426 if len(job.CachePaths) == 0 {
427 return nil
428 }
429 root := r.cacheDir(repoID, job.CacheKey)
430 for _, rel := range job.CachePaths {
431 src := filepath.Join(root, filepath.FromSlash(rel))
432 if _, err := os.Stat(src); err != nil {
433 continue
434 }
435 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
436 return err
437 }
438 }
439 return nil
440}
441
442func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
443 if len(job.CachePaths) == 0 {
444 return nil
445 }
446 root := r.cacheDir(repoID, job.CacheKey)
447 for _, rel := range job.CachePaths {
448 src := filepath.Join(work, filepath.FromSlash(rel))
449 info, err := os.Stat(src)
450 if err != nil || info.IsDir() {
451 continue
452 }
453 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
454 return err
455 }
456 }
457 return nil
458}
459
460func (r *Runner) cacheDir(repoID int64, key string) string {
461 if key == "" {
462 key = "default"
463 }
464 var b strings.Builder
465 for _, r := range key {
466 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
467 b.WriteRune(r)
468 }
469 }
470 safe := b.String()
471 if safe == "" {
472 safe = "default"
473 }
474 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
475}
476
477func stamp(line string) string {
478 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
479}
480
481type logSink struct {
482 mu sync.Mutex
483 store *store.Store
484 jobID int64
485 secrets []string
486 partial string
487 current string
488 opened time.Time
489}
490
491func (l *logSink) Write(p []byte) (int, error) {
492 l.mu.Lock()
493 defer l.mu.Unlock()
494 l.partial += string(p)
495 for {
496 i := strings.IndexByte(l.partial, '\n')
497 if i < 0 {
498 break
499 }
500 l.emit(strings.TrimRight(l.partial[:i], "\r"))
501 l.partial = l.partial[i+1:]
502 }
503 return len(p), nil
504}
505
506func (l *logSink) flush() {
507 l.mu.Lock()
508 defer l.mu.Unlock()
509 if strings.TrimSpace(l.partial) != "" {
510 l.emit(l.partial)
511 }
512 l.partial = ""
513}
514
515func (l *logSink) emit(line string) {
516 line = ci.Mask(line, l.secrets)
517 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
518 if l.current != "" && strings.HasPrefix(line, "section_end:") {
519 dur := int(time.Since(l.opened).Seconds())
520 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
521 l.current = ""
522 }
523 if strings.HasPrefix(line, "section_start:") {
524 l.current = line
525 l.opened = time.Now()
526 }
527 }
528 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
529}
530
531func (l *logSink) section(name, title string) {
532 l.emit("section_start:" + name + ":" + title)
533}
534
535func (l *logSink) closeSection() {
536 if l.current == "" {
537 return
538 }
539 name := "prepare"
540 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
541 name = parts[0]
542 }
543 l.emit("section_end:" + name + ":done")
544}
545
546func copyFile(src, dst string) error {
547 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
548 return err
549 }
550 in, err := os.Open(src)
551 if err != nil {
552 return err
553 }
554 defer in.Close()
555 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
556 if err != nil {
557 return err
558 }
559 defer out.Close()
560 _, err = io.Copy(out, in)
561 return err
562}
563
564func execOut(ctx context.Context, bin string, args ...string) (string, error) {
565 if bin == "" {
566 bin = "git"
567 }
568 cmd := exec.CommandContext(ctx, bin, args...)
569 var stdout, stderr bytes.Buffer
570 cmd.Stdout = &stdout
571 cmd.Stderr = &stderr
572 if err := cmd.Run(); err != nil {
573 msg := strings.TrimSpace(stderr.String())
574 if msg == "" {
575 msg = err.Error()
576 }
577 return "", fmt.Errorf("%s", msg)
578 }
579 return stdout.String(), nil
580}