BlueGit

timfa bluegit

Files

bluegit / internal / ci / parse.go

parse.go

feat: eerste basis van BlueGit
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
151aa352 History
parse.go 13.4 KiB Download Raw
1// Package ci leest blue-ci-cd.yml en bepaalt welke jobs draaien.
2// De vorm ligt dicht bij GitLab CI: stages, jobs, needs, rules, artifacts.
3package ci
4
5import (
6 "fmt"
7 "regexp"
8 "strconv"
9 "strings"
10 "time"
11
12 "gopkg.in/yaml.v3"
13)
14
15// Pipeline is een geparset configuratiebestand.
16type Pipeline struct {
17 Stages []string
18 Variables map[string]string
19 Image string
20 Jobs []Job
21}
22
23// Job is één containerstap.
24type Job struct {
25 Name string
26 Stage string
27 Image string
28 Script []string
29 Needs []string
30 NeedsSet bool
31 Variables map[string]string
32 Artifacts Artifacts
33 Rules []Rule
34 AllowFailure bool
35}
36
37// Artifacts zijn paden die na een geslaagde job bewaard blijven.
38type Artifacts struct {
39 Paths []string
40 ExpireIn time.Duration
41}
42
43// Rule is één voorwaarde. De job draait als minstens één rule klopt.
44// Zonder rules draait de job altijd.
45type Rule struct {
46 If string
47}
48
49// Parse leest YAML. Jobvolgorde blijft de volgorde uit het bestand.
50func Parse(data []byte) (Pipeline, error) {
51 var root yaml.Node
52 if err := yaml.Unmarshal(data, &root); err != nil {
53 return Pipeline{}, fmt.Errorf("yaml: %w", err)
54 }
55 if root.Kind != yaml.DocumentNode || len(root.Content) != 1 || root.Content[0].Kind != yaml.MappingNode {
56 return Pipeline{}, fmt.Errorf("blue-ci-cd.yml moet een mapping zijn")
57 }
58 body := root.Content[0]
59 var p Pipeline
60 p.Variables = map[string]string{}
61 for i := 0; i < len(body.Content); i += 2 {
62 key := body.Content[i].Value
63 val := body.Content[i+1]
64 switch key {
65 case "stages":
66 stages, err := scalarList(val)
67 if err != nil {
68 return Pipeline{}, fmt.Errorf("stages: %w", err)
69 }
70 p.Stages = stages
71 case "variables":
72 vars, err := scalarMap(val)
73 if err != nil {
74 return Pipeline{}, fmt.Errorf("variables: %w", err)
75 }
76 p.Variables = vars
77 case "image":
78 if val.Kind != yaml.ScalarNode {
79 return Pipeline{}, fmt.Errorf("image moet een tekst zijn")
80 }
81 p.Image = val.Value
82 case "jobs":
83 jobs, err := parseJobs(val)
84 if err != nil {
85 return Pipeline{}, err
86 }
87 p.Jobs = jobs
88 default:
89 return Pipeline{}, fmt.Errorf("onbekend veld %q", key)
90 }
91 }
92 if err := p.validate(); err != nil {
93 return Pipeline{}, err
94 }
95 return p, nil
96}
97
98func parseJobs(node *yaml.Node) ([]Job, error) {
99 if node.Kind != yaml.MappingNode {
100 return nil, fmt.Errorf("jobs moet een mapping zijn")
101 }
102 var jobs []Job
103 for i := 0; i < len(node.Content); i += 2 {
104 name := node.Content[i].Value
105 body := node.Content[i+1]
106 job, err := parseJob(name, body)
107 if err != nil {
108 return nil, fmt.Errorf("job %s: %w", name, err)
109 }
110 jobs = append(jobs, job)
111 }
112 return jobs, nil
113}
114
115func parseJob(name string, node *yaml.Node) (Job, error) {
116 if node.Kind != yaml.MappingNode {
117 return Job{}, fmt.Errorf("moet een mapping zijn")
118 }
119 j := Job{Name: name, Variables: map[string]string{}}
120 for i := 0; i < len(node.Content); i += 2 {
121 key := node.Content[i].Value
122 val := node.Content[i+1]
123 switch key {
124 case "stage":
125 if val.Kind != yaml.ScalarNode {
126 return Job{}, fmt.Errorf("stage moet tekst zijn")
127 }
128 j.Stage = val.Value
129 case "image":
130 if val.Kind != yaml.ScalarNode {
131 return Job{}, fmt.Errorf("image moet tekst zijn")
132 }
133 j.Image = val.Value
134 case "script":
135 lines, err := scalarList(val)
136 if err != nil {
137 return Job{}, fmt.Errorf("script: %w", err)
138 }
139 j.Script = lines
140 case "needs":
141 needs, err := scalarList(val)
142 if err != nil {
143 return Job{}, fmt.Errorf("needs: %w", err)
144 }
145 j.Needs = needs
146 j.NeedsSet = true
147 case "variables":
148 vars, err := scalarMap(val)
149 if err != nil {
150 return Job{}, fmt.Errorf("variables: %w", err)
151 }
152 j.Variables = vars
153 case "artifacts":
154 art, err := parseArtifacts(val)
155 if err != nil {
156 return Job{}, err
157 }
158 j.Artifacts = art
159 case "rules":
160 rules, err := parseRules(val)
161 if err != nil {
162 return Job{}, err
163 }
164 j.Rules = rules
165 case "allow_failure":
166 b, err := parseBool(val)
167 if err != nil {
168 return Job{}, fmt.Errorf("allow_failure: %w", err)
169 }
170 j.AllowFailure = b
171 default:
172 return Job{}, fmt.Errorf("onbekend veld %q", key)
173 }
174 }
175 return j, nil
176}
177
178func parseArtifacts(node *yaml.Node) (Artifacts, error) {
179 if node.Kind != yaml.MappingNode {
180 return Artifacts{}, fmt.Errorf("artifacts moet een mapping zijn")
181 }
182 var a Artifacts
183 for i := 0; i < len(node.Content); i += 2 {
184 key := node.Content[i].Value
185 val := node.Content[i+1]
186 switch key {
187 case "paths":
188 paths, err := scalarList(val)
189 if err != nil {
190 return Artifacts{}, fmt.Errorf("artifacts.paths: %w", err)
191 }
192 a.Paths = paths
193 case "expire_in":
194 if val.Kind != yaml.ScalarNode {
195 return Artifacts{}, fmt.Errorf("expire_in moet tekst zijn")
196 }
197 d, err := parseExpire(val.Value)
198 if err != nil {
199 return Artifacts{}, err
200 }
201 a.ExpireIn = d
202 default:
203 return Artifacts{}, fmt.Errorf("onbekend artifacts-veld %q", key)
204 }
205 }
206 return a, nil
207}
208
209func parseRules(node *yaml.Node) ([]Rule, error) {
210 if node.Kind != yaml.SequenceNode {
211 return nil, fmt.Errorf("rules moet een lijst zijn")
212 }
213 var rules []Rule
214 for _, item := range node.Content {
215 if item.Kind != yaml.MappingNode {
216 return nil, fmt.Errorf("rule moet een mapping zijn")
217 }
218 var r Rule
219 for i := 0; i < len(item.Content); i += 2 {
220 key := item.Content[i].Value
221 val := item.Content[i+1]
222 if key != "if" {
223 return nil, fmt.Errorf("rule ondersteunt alleen if, kreeg %q", key)
224 }
225 if val.Kind != yaml.ScalarNode || strings.TrimSpace(val.Value) == "" {
226 return nil, fmt.Errorf("rule.if moet tekst zijn")
227 }
228 r.If = strings.TrimSpace(val.Value)
229 }
230 if r.If == "" {
231 return nil, fmt.Errorf("rule mist if")
232 }
233 if _, err := compileRule(r.If); err != nil {
234 return nil, err
235 }
236 rules = append(rules, r)
237 }
238 return rules, nil
239}
240
241func (p *Pipeline) validate() error {
242 if len(p.Jobs) == 0 {
243 return fmt.Errorf("geen jobs")
244 }
245 seenStage := map[string]bool{}
246 for _, s := range p.Stages {
247 if s == "" {
248 return fmt.Errorf("lege stage-naam")
249 }
250 if seenStage[s] {
251 return fmt.Errorf("dubbele stage %q", s)
252 }
253 seenStage[s] = true
254 }
255 if len(p.Stages) == 0 {
256 var order []string
257 have := map[string]bool{}
258 for i := range p.Jobs {
259 if p.Jobs[i].Stage == "" {
260 p.Jobs[i].Stage = "test"
261 }
262 if !have[p.Jobs[i].Stage] {
263 have[p.Jobs[i].Stage] = true
264 order = append(order, p.Jobs[i].Stage)
265 }
266 }
267 p.Stages = order
268 for _, s := range p.Stages {
269 seenStage[s] = true
270 }
271 }
272 names := map[string]bool{}
273 for i := range p.Jobs {
274 j := &p.Jobs[i]
275 if j.Name == "" || strings.ContainsAny(j.Name, " \t\n/\\") {
276 return fmt.Errorf("ongeldige jobnaam %q", j.Name)
277 }
278 if names[j.Name] {
279 return fmt.Errorf("dubbele job %q", j.Name)
280 }
281 names[j.Name] = true
282 if j.Stage == "" {
283 return fmt.Errorf("job %s mist stage", j.Name)
284 }
285 if !seenStage[j.Stage] {
286 return fmt.Errorf("job %s gebruikt onbekende stage %q", j.Name, j.Stage)
287 }
288 if j.Image == "" {
289 j.Image = p.Image
290 }
291 if strings.TrimSpace(j.Image) == "" || strings.ContainsAny(j.Image, " \t\n") {
292 return fmt.Errorf("job %s mist een image", j.Name)
293 }
294 if len(j.Script) == 0 {
295 return fmt.Errorf("job %s mist script", j.Name)
296 }
297 for _, line := range j.Script {
298 if strings.TrimSpace(line) == "" {
299 return fmt.Errorf("job %s heeft een lege scriptregel", j.Name)
300 }
301 }
302 for _, path := range j.Artifacts.Paths {
303 if err := safeRelPath(path); err != nil {
304 return fmt.Errorf("job %s artifact: %w", j.Name, err)
305 }
306 }
307 }
308 for _, j := range p.Jobs {
309 for _, need := range j.Needs {
310 if !names[need] {
311 return fmt.Errorf("job %s needs onbekende job %q", j.Name, need)
312 }
313 if need == j.Name {
314 return fmt.Errorf("job %s kan niet van zichzelf afhangen", j.Name)
315 }
316 }
317 }
318 if err := detectCycles(p.Jobs); err != nil {
319 return err
320 }
321 return nil
322}
323
324func detectCycles(jobs []Job) error {
325 idx := map[string]int{}
326 for i, j := range jobs {
327 idx[j.Name] = i
328 }
329 const (
330 white = 0
331 gray = 1
332 black = 2
333 )
334 color := make([]int, len(jobs))
335 var visit func(int) error
336 visit = func(i int) error {
337 if color[i] == gray {
338 return fmt.Errorf("cyclus in needs bij job %s", jobs[i].Name)
339 }
340 if color[i] == black {
341 return nil
342 }
343 color[i] = gray
344 for _, n := range jobs[i].Needs {
345 if err := visit(idx[n]); err != nil {
346 return err
347 }
348 }
349 color[i] = black
350 return nil
351 }
352 for i := range jobs {
353 if err := visit(i); err != nil {
354 return err
355 }
356 }
357 return nil
358}
359
360// ActiveJobs filtert jobs waarvan de rules niet kloppen.
361func (p Pipeline) ActiveJobs(env map[string]string) ([]Job, error) {
362 var out []Job
363 for _, j := range p.Jobs {
364 ok, err := j.matches(env)
365 if err != nil {
366 return nil, fmt.Errorf("job %s: %w", j.Name, err)
367 }
368 if ok {
369 out = append(out, j)
370 }
371 }
372 return out, nil
373}
374
375func (j Job) matches(env map[string]string) (bool, error) {
376 if len(j.Rules) == 0 {
377 return true, nil
378 }
379 for _, r := range j.Rules {
380 ok, err := evalRule(r.If, env)
381 if err != nil {
382 return false, err
383 }
384 if ok {
385 return true, nil
386 }
387 }
388 return false, nil
389}
390
391// Gate beslist of een job mag starten.
392// "ready" betekent dat alle voorgangers klaar zijn.
393// "wait" betekent dat er nog een voorganger loopt of in de wachtrij staat.
394// "skip" betekent dat een voorganger mislukte zonder allow_failure.
395//
396// NeedsSet volgt de genoemde jobs, ook als de lijst leeg is.
397// Zonder needs wacht de job op alle jobs in eerdere stages.
398func Gate(jobs []Job, status map[string]string, job Job) string {
399 byName := map[string]Job{}
400 for _, j := range jobs {
401 byName[j.Name] = j
402 }
403 var deps []Job
404 if job.NeedsSet {
405 for _, n := range job.Needs {
406 deps = append(deps, byName[n])
407 }
408 } else {
409 stagePos := map[string]int{}
410 order := 0
411 for _, j := range jobs {
412 if _, ok := stagePos[j.Stage]; !ok {
413 stagePos[j.Stage] = order
414 order++
415 }
416 }
417 my := stagePos[job.Stage]
418 for _, j := range jobs {
419 if stagePos[j.Stage] < my {
420 deps = append(deps, j)
421 }
422 }
423 }
424 for _, d := range deps {
425 switch status[d.Name] {
426 case "success", "skipped":
427 continue
428 case "failed":
429 if d.AllowFailure {
430 continue
431 }
432 return "skip"
433 default:
434 return "wait"
435 }
436 }
437 return "ready"
438}
439
440var (
441 reEq = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*)\s*==\s*"([^"]*)"$`)
442 reNe = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*)\s*!=\s*"([^"]*)"$`)
443 reRe = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*)\s*=~\s*/(.+)/$`)
444)
445
446type compiledRule struct {
447 kind string
448 name string
449 arg string
450 re *regexp.Regexp
451}
452
453func compileRule(expr string) (compiledRule, error) {
454 if m := reEq.FindStringSubmatch(expr); m != nil {
455 return compiledRule{kind: "eq", name: m[1], arg: m[2]}, nil
456 }
457 if m := reNe.FindStringSubmatch(expr); m != nil {
458 return compiledRule{kind: "ne", name: m[1], arg: m[2]}, nil
459 }
460 if m := reRe.FindStringSubmatch(expr); m != nil {
461 re, err := regexp.Compile(m[2])
462 if err != nil {
463 return compiledRule{}, fmt.Errorf("ongeldige regex in rule %q: %w", expr, err)
464 }
465 return compiledRule{kind: "re", name: m[1], re: re}, nil
466 }
467 return compiledRule{}, fmt.Errorf("rule %q wordt niet begrepen", expr)
468}
469
470func evalRule(expr string, env map[string]string) (bool, error) {
471 c, err := compileRule(expr)
472 if err != nil {
473 return false, err
474 }
475 val := env[c.name]
476 switch c.kind {
477 case "eq":
478 return val == c.arg, nil
479 case "ne":
480 return val != c.arg, nil
481 case "re":
482 return c.re.MatchString(val), nil
483 default:
484 return false, fmt.Errorf("interne rulefout")
485 }
486}
487
488func parseExpire(s string) (time.Duration, error) {
489 s = strings.TrimSpace(s)
490 if s == "" || s == "never" {
491 return 0, nil
492 }
493 if len(s) < 2 {
494 return 0, fmt.Errorf("expire_in %q is ongeldig", s)
495 }
496 n, err := strconv.Atoi(s[:len(s)-1])
497 if err != nil || n < 0 {
498 return 0, fmt.Errorf("expire_in %q is ongeldig", s)
499 }
500 switch s[len(s)-1] {
501 case 'm':
502 return time.Duration(n) * time.Minute, nil
503 case 'h':
504 return time.Duration(n) * time.Hour, nil
505 case 'd':
506 return time.Duration(n) * 24 * time.Hour, nil
507 case 'w':
508 return time.Duration(n) * 7 * 24 * time.Hour, nil
509 default:
510 return 0, fmt.Errorf("expire_in %q is ongeldig", s)
511 }
512}
513
514func scalarList(node *yaml.Node) ([]string, error) {
515 if node.Kind != yaml.SequenceNode {
516 return nil, fmt.Errorf("verwacht een lijst")
517 }
518 var out []string
519 for _, item := range node.Content {
520 if item.Kind != yaml.ScalarNode {
521 return nil, fmt.Errorf("lijst mag alleen tekst bevatten")
522 }
523 out = append(out, item.Value)
524 }
525 return out, nil
526}
527
528func scalarMap(node *yaml.Node) (map[string]string, error) {
529 if node.Kind != yaml.MappingNode {
530 return nil, fmt.Errorf("verwacht een mapping")
531 }
532 out := map[string]string{}
533 for i := 0; i < len(node.Content); i += 2 {
534 k := node.Content[i]
535 v := node.Content[i+1]
536 if k.Kind != yaml.ScalarNode || v.Kind != yaml.ScalarNode {
537 return nil, fmt.Errorf("variabelen moeten tekst zijn")
538 }
539 out[k.Value] = v.Value
540 }
541 return out, nil
542}
543
544func parseBool(node *yaml.Node) (bool, error) {
545 if node.Kind != yaml.ScalarNode {
546 return false, fmt.Errorf("verwacht ja of nee")
547 }
548 switch strings.ToLower(node.Value) {
549 case "true", "yes", "ja":
550 return true, nil
551 case "false", "no", "nee":
552 return false, nil
553 default:
554 return false, fmt.Errorf("verwacht true of false")
555 }
556}
557
558func safeRelPath(p string) error {
559 if p == "" || strings.HasPrefix(p, "/") || strings.Contains(p, "\\") {
560 return fmt.Errorf("pad %q moet relatief zijn", p)
561 }
562 for _, part := range strings.Split(p, "/") {
563 if part == ".." {
564 return fmt.Errorf("pad %q mag niet buiten de workspace", p)
565 }
566 }
567 return nil
568}