| 1 | // Package ci leest blue-ci-cd.yml en bepaalt welke jobs draaien. |
| 2 | // De vorm ligt dicht bij GitLab CI: stages, jobs, needs, rules, artifacts. |
| 3 | package ci |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "regexp" |
| 8 | "strconv" |
| 9 | "strings" |
| 10 | "time" |
| 11 | |
| 12 | "gopkg.in/yaml.v3" |
| 13 | ) |
| 14 | |
| 15 | // Pipeline is een geparset configuratiebestand. |
| 16 | type Pipeline struct { |
| 17 | Stages []string |
| 18 | Variables map[string]string |
| 19 | Image string |
| 20 | Jobs []Job |
| 21 | } |
| 22 | |
| 23 | // Job is één containerstap. |
| 24 | type Job struct { |
| 25 | Name string |
| 26 | Stage string |
| 27 | Image string |
| 28 | Script []string |
| 29 | Needs []string |
| 30 | NeedsSet bool |
| 31 | Variables map[string]string |
| 32 | Artifacts Artifacts |
| 33 | Rules []Rule |
| 34 | AllowFailure bool |
| 35 | } |
| 36 | |
| 37 | // Artifacts zijn paden die na een geslaagde job bewaard blijven. |
| 38 | type Artifacts struct { |
| 39 | Paths []string |
| 40 | ExpireIn time.Duration |
| 41 | } |
| 42 | |
| 43 | // Rule is één voorwaarde. De job draait als minstens één rule klopt. |
| 44 | // Zonder rules draait de job altijd. |
| 45 | type Rule struct { |
| 46 | If string |
| 47 | } |
| 48 | |
| 49 | // Parse leest YAML. Jobvolgorde blijft de volgorde uit het bestand. |
| 50 | func Parse(data []byte) (Pipeline, error) { |
| 51 | var root yaml.Node |
| 52 | if err := yaml.Unmarshal(data, &root); err != nil { |
| 53 | return Pipeline{}, fmt.Errorf("yaml: %w", err) |
| 54 | } |
| 55 | if root.Kind != yaml.DocumentNode || len(root.Content) != 1 || root.Content[0].Kind != yaml.MappingNode { |
| 56 | return Pipeline{}, fmt.Errorf("blue-ci-cd.yml moet een mapping zijn") |
| 57 | } |
| 58 | body := root.Content[0] |
| 59 | var p Pipeline |
| 60 | p.Variables = map[string]string{} |
| 61 | for i := 0; i < len(body.Content); i += 2 { |
| 62 | key := body.Content[i].Value |
| 63 | val := body.Content[i+1] |
| 64 | switch key { |
| 65 | case "stages": |
| 66 | stages, err := scalarList(val) |
| 67 | if err != nil { |
| 68 | return Pipeline{}, fmt.Errorf("stages: %w", err) |
| 69 | } |
| 70 | p.Stages = stages |
| 71 | case "variables": |
| 72 | vars, err := scalarMap(val) |
| 73 | if err != nil { |
| 74 | return Pipeline{}, fmt.Errorf("variables: %w", err) |
| 75 | } |
| 76 | p.Variables = vars |
| 77 | case "image": |
| 78 | if val.Kind != yaml.ScalarNode { |
| 79 | return Pipeline{}, fmt.Errorf("image moet een tekst zijn") |
| 80 | } |
| 81 | p.Image = val.Value |
| 82 | case "jobs": |
| 83 | jobs, err := parseJobs(val) |
| 84 | if err != nil { |
| 85 | return Pipeline{}, err |
| 86 | } |
| 87 | p.Jobs = jobs |
| 88 | default: |
| 89 | return Pipeline{}, fmt.Errorf("onbekend veld %q", key) |
| 90 | } |
| 91 | } |
| 92 | if err := p.validate(); err != nil { |
| 93 | return Pipeline{}, err |
| 94 | } |
| 95 | return p, nil |
| 96 | } |
| 97 | |
| 98 | func parseJobs(node *yaml.Node) ([]Job, error) { |
| 99 | if node.Kind != yaml.MappingNode { |
| 100 | return nil, fmt.Errorf("jobs moet een mapping zijn") |
| 101 | } |
| 102 | var jobs []Job |
| 103 | for i := 0; i < len(node.Content); i += 2 { |
| 104 | name := node.Content[i].Value |
| 105 | body := node.Content[i+1] |
| 106 | job, err := parseJob(name, body) |
| 107 | if err != nil { |
| 108 | return nil, fmt.Errorf("job %s: %w", name, err) |
| 109 | } |
| 110 | jobs = append(jobs, job) |
| 111 | } |
| 112 | return jobs, nil |
| 113 | } |
| 114 | |
| 115 | func parseJob(name string, node *yaml.Node) (Job, error) { |
| 116 | if node.Kind != yaml.MappingNode { |
| 117 | return Job{}, fmt.Errorf("moet een mapping zijn") |
| 118 | } |
| 119 | j := Job{Name: name, Variables: map[string]string{}} |
| 120 | for i := 0; i < len(node.Content); i += 2 { |
| 121 | key := node.Content[i].Value |
| 122 | val := node.Content[i+1] |
| 123 | switch key { |
| 124 | case "stage": |
| 125 | if val.Kind != yaml.ScalarNode { |
| 126 | return Job{}, fmt.Errorf("stage moet tekst zijn") |
| 127 | } |
| 128 | j.Stage = val.Value |
| 129 | case "image": |
| 130 | if val.Kind != yaml.ScalarNode { |
| 131 | return Job{}, fmt.Errorf("image moet tekst zijn") |
| 132 | } |
| 133 | j.Image = val.Value |
| 134 | case "script": |
| 135 | lines, err := scalarList(val) |
| 136 | if err != nil { |
| 137 | return Job{}, fmt.Errorf("script: %w", err) |
| 138 | } |
| 139 | j.Script = lines |
| 140 | case "needs": |
| 141 | needs, err := scalarList(val) |
| 142 | if err != nil { |
| 143 | return Job{}, fmt.Errorf("needs: %w", err) |
| 144 | } |
| 145 | j.Needs = needs |
| 146 | j.NeedsSet = true |
| 147 | case "variables": |
| 148 | vars, err := scalarMap(val) |
| 149 | if err != nil { |
| 150 | return Job{}, fmt.Errorf("variables: %w", err) |
| 151 | } |
| 152 | j.Variables = vars |
| 153 | case "artifacts": |
| 154 | art, err := parseArtifacts(val) |
| 155 | if err != nil { |
| 156 | return Job{}, err |
| 157 | } |
| 158 | j.Artifacts = art |
| 159 | case "rules": |
| 160 | rules, err := parseRules(val) |
| 161 | if err != nil { |
| 162 | return Job{}, err |
| 163 | } |
| 164 | j.Rules = rules |
| 165 | case "allow_failure": |
| 166 | b, err := parseBool(val) |
| 167 | if err != nil { |
| 168 | return Job{}, fmt.Errorf("allow_failure: %w", err) |
| 169 | } |
| 170 | j.AllowFailure = b |
| 171 | default: |
| 172 | return Job{}, fmt.Errorf("onbekend veld %q", key) |
| 173 | } |
| 174 | } |
| 175 | return j, nil |
| 176 | } |
| 177 | |
| 178 | func parseArtifacts(node *yaml.Node) (Artifacts, error) { |
| 179 | if node.Kind != yaml.MappingNode { |
| 180 | return Artifacts{}, fmt.Errorf("artifacts moet een mapping zijn") |
| 181 | } |
| 182 | var a Artifacts |
| 183 | for i := 0; i < len(node.Content); i += 2 { |
| 184 | key := node.Content[i].Value |
| 185 | val := node.Content[i+1] |
| 186 | switch key { |
| 187 | case "paths": |
| 188 | paths, err := scalarList(val) |
| 189 | if err != nil { |
| 190 | return Artifacts{}, fmt.Errorf("artifacts.paths: %w", err) |
| 191 | } |
| 192 | a.Paths = paths |
| 193 | case "expire_in": |
| 194 | if val.Kind != yaml.ScalarNode { |
| 195 | return Artifacts{}, fmt.Errorf("expire_in moet tekst zijn") |
| 196 | } |
| 197 | d, err := parseExpire(val.Value) |
| 198 | if err != nil { |
| 199 | return Artifacts{}, err |
| 200 | } |
| 201 | a.ExpireIn = d |
| 202 | default: |
| 203 | return Artifacts{}, fmt.Errorf("onbekend artifacts-veld %q", key) |
| 204 | } |
| 205 | } |
| 206 | return a, nil |
| 207 | } |
| 208 | |
| 209 | func parseRules(node *yaml.Node) ([]Rule, error) { |
| 210 | if node.Kind != yaml.SequenceNode { |
| 211 | return nil, fmt.Errorf("rules moet een lijst zijn") |
| 212 | } |
| 213 | var rules []Rule |
| 214 | for _, item := range node.Content { |
| 215 | if item.Kind != yaml.MappingNode { |
| 216 | return nil, fmt.Errorf("rule moet een mapping zijn") |
| 217 | } |
| 218 | var r Rule |
| 219 | for i := 0; i < len(item.Content); i += 2 { |
| 220 | key := item.Content[i].Value |
| 221 | val := item.Content[i+1] |
| 222 | if key != "if" { |
| 223 | return nil, fmt.Errorf("rule ondersteunt alleen if, kreeg %q", key) |
| 224 | } |
| 225 | if val.Kind != yaml.ScalarNode || strings.TrimSpace(val.Value) == "" { |
| 226 | return nil, fmt.Errorf("rule.if moet tekst zijn") |
| 227 | } |
| 228 | r.If = strings.TrimSpace(val.Value) |
| 229 | } |
| 230 | if r.If == "" { |
| 231 | return nil, fmt.Errorf("rule mist if") |
| 232 | } |
| 233 | if _, err := compileRule(r.If); err != nil { |
| 234 | return nil, err |
| 235 | } |
| 236 | rules = append(rules, r) |
| 237 | } |
| 238 | return rules, nil |
| 239 | } |
| 240 | |
| 241 | func (p *Pipeline) validate() error { |
| 242 | if len(p.Jobs) == 0 { |
| 243 | return fmt.Errorf("geen jobs") |
| 244 | } |
| 245 | seenStage := map[string]bool{} |
| 246 | for _, s := range p.Stages { |
| 247 | if s == "" { |
| 248 | return fmt.Errorf("lege stage-naam") |
| 249 | } |
| 250 | if seenStage[s] { |
| 251 | return fmt.Errorf("dubbele stage %q", s) |
| 252 | } |
| 253 | seenStage[s] = true |
| 254 | } |
| 255 | if len(p.Stages) == 0 { |
| 256 | var order []string |
| 257 | have := map[string]bool{} |
| 258 | for i := range p.Jobs { |
| 259 | if p.Jobs[i].Stage == "" { |
| 260 | p.Jobs[i].Stage = "test" |
| 261 | } |
| 262 | if !have[p.Jobs[i].Stage] { |
| 263 | have[p.Jobs[i].Stage] = true |
| 264 | order = append(order, p.Jobs[i].Stage) |
| 265 | } |
| 266 | } |
| 267 | p.Stages = order |
| 268 | for _, s := range p.Stages { |
| 269 | seenStage[s] = true |
| 270 | } |
| 271 | } |
| 272 | names := map[string]bool{} |
| 273 | for i := range p.Jobs { |
| 274 | j := &p.Jobs[i] |
| 275 | if j.Name == "" || strings.ContainsAny(j.Name, " \t\n/\\") { |
| 276 | return fmt.Errorf("ongeldige jobnaam %q", j.Name) |
| 277 | } |
| 278 | if names[j.Name] { |
| 279 | return fmt.Errorf("dubbele job %q", j.Name) |
| 280 | } |
| 281 | names[j.Name] = true |
| 282 | if j.Stage == "" { |
| 283 | return fmt.Errorf("job %s mist stage", j.Name) |
| 284 | } |
| 285 | if !seenStage[j.Stage] { |
| 286 | return fmt.Errorf("job %s gebruikt onbekende stage %q", j.Name, j.Stage) |
| 287 | } |
| 288 | if j.Image == "" { |
| 289 | j.Image = p.Image |
| 290 | } |
| 291 | if strings.TrimSpace(j.Image) == "" || strings.ContainsAny(j.Image, " \t\n") { |
| 292 | return fmt.Errorf("job %s mist een image", j.Name) |
| 293 | } |
| 294 | if len(j.Script) == 0 { |
| 295 | return fmt.Errorf("job %s mist script", j.Name) |
| 296 | } |
| 297 | for _, line := range j.Script { |
| 298 | if strings.TrimSpace(line) == "" { |
| 299 | return fmt.Errorf("job %s heeft een lege scriptregel", j.Name) |
| 300 | } |
| 301 | } |
| 302 | for _, path := range j.Artifacts.Paths { |
| 303 | if err := safeRelPath(path); err != nil { |
| 304 | return fmt.Errorf("job %s artifact: %w", j.Name, err) |
| 305 | } |
| 306 | } |
| 307 | } |
| 308 | for _, j := range p.Jobs { |
| 309 | for _, need := range j.Needs { |
| 310 | if !names[need] { |
| 311 | return fmt.Errorf("job %s needs onbekende job %q", j.Name, need) |
| 312 | } |
| 313 | if need == j.Name { |
| 314 | return fmt.Errorf("job %s kan niet van zichzelf afhangen", j.Name) |
| 315 | } |
| 316 | } |
| 317 | } |
| 318 | if err := detectCycles(p.Jobs); err != nil { |
| 319 | return err |
| 320 | } |
| 321 | return nil |
| 322 | } |
| 323 | |
| 324 | func detectCycles(jobs []Job) error { |
| 325 | idx := map[string]int{} |
| 326 | for i, j := range jobs { |
| 327 | idx[j.Name] = i |
| 328 | } |
| 329 | const ( |
| 330 | white = 0 |
| 331 | gray = 1 |
| 332 | black = 2 |
| 333 | ) |
| 334 | color := make([]int, len(jobs)) |
| 335 | var visit func(int) error |
| 336 | visit = func(i int) error { |
| 337 | if color[i] == gray { |
| 338 | return fmt.Errorf("cyclus in needs bij job %s", jobs[i].Name) |
| 339 | } |
| 340 | if color[i] == black { |
| 341 | return nil |
| 342 | } |
| 343 | color[i] = gray |
| 344 | for _, n := range jobs[i].Needs { |
| 345 | if err := visit(idx[n]); err != nil { |
| 346 | return err |
| 347 | } |
| 348 | } |
| 349 | color[i] = black |
| 350 | return nil |
| 351 | } |
| 352 | for i := range jobs { |
| 353 | if err := visit(i); err != nil { |
| 354 | return err |
| 355 | } |
| 356 | } |
| 357 | return nil |
| 358 | } |
| 359 | |
| 360 | // ActiveJobs filtert jobs waarvan de rules niet kloppen. |
| 361 | func (p Pipeline) ActiveJobs(env map[string]string) ([]Job, error) { |
| 362 | var out []Job |
| 363 | for _, j := range p.Jobs { |
| 364 | ok, err := j.matches(env) |
| 365 | if err != nil { |
| 366 | return nil, fmt.Errorf("job %s: %w", j.Name, err) |
| 367 | } |
| 368 | if ok { |
| 369 | out = append(out, j) |
| 370 | } |
| 371 | } |
| 372 | return out, nil |
| 373 | } |
| 374 | |
| 375 | func (j Job) matches(env map[string]string) (bool, error) { |
| 376 | if len(j.Rules) == 0 { |
| 377 | return true, nil |
| 378 | } |
| 379 | for _, r := range j.Rules { |
| 380 | ok, err := evalRule(r.If, env) |
| 381 | if err != nil { |
| 382 | return false, err |
| 383 | } |
| 384 | if ok { |
| 385 | return true, nil |
| 386 | } |
| 387 | } |
| 388 | return false, nil |
| 389 | } |
| 390 | |
| 391 | // Gate beslist of een job mag starten. |
| 392 | // "ready" betekent dat alle voorgangers klaar zijn. |
| 393 | // "wait" betekent dat er nog een voorganger loopt of in de wachtrij staat. |
| 394 | // "skip" betekent dat een voorganger mislukte zonder allow_failure. |
| 395 | // |
| 396 | // NeedsSet volgt de genoemde jobs, ook als de lijst leeg is. |
| 397 | // Zonder needs wacht de job op alle jobs in eerdere stages. |
| 398 | func Gate(jobs []Job, status map[string]string, job Job) string { |
| 399 | byName := map[string]Job{} |
| 400 | for _, j := range jobs { |
| 401 | byName[j.Name] = j |
| 402 | } |
| 403 | var deps []Job |
| 404 | if job.NeedsSet { |
| 405 | for _, n := range job.Needs { |
| 406 | deps = append(deps, byName[n]) |
| 407 | } |
| 408 | } else { |
| 409 | stagePos := map[string]int{} |
| 410 | order := 0 |
| 411 | for _, j := range jobs { |
| 412 | if _, ok := stagePos[j.Stage]; !ok { |
| 413 | stagePos[j.Stage] = order |
| 414 | order++ |
| 415 | } |
| 416 | } |
| 417 | my := stagePos[job.Stage] |
| 418 | for _, j := range jobs { |
| 419 | if stagePos[j.Stage] < my { |
| 420 | deps = append(deps, j) |
| 421 | } |
| 422 | } |
| 423 | } |
| 424 | for _, d := range deps { |
| 425 | switch status[d.Name] { |
| 426 | case "success", "skipped": |
| 427 | continue |
| 428 | case "failed": |
| 429 | if d.AllowFailure { |
| 430 | continue |
| 431 | } |
| 432 | return "skip" |
| 433 | default: |
| 434 | return "wait" |
| 435 | } |
| 436 | } |
| 437 | return "ready" |
| 438 | } |
| 439 | |
| 440 | var ( |
| 441 | reEq = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*)\s*==\s*"([^"]*)"$`) |
| 442 | reNe = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*)\s*!=\s*"([^"]*)"$`) |
| 443 | reRe = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*)\s*=~\s*/(.+)/$`) |
| 444 | ) |
| 445 | |
| 446 | type compiledRule struct { |
| 447 | kind string |
| 448 | name string |
| 449 | arg string |
| 450 | re *regexp.Regexp |
| 451 | } |
| 452 | |
| 453 | func compileRule(expr string) (compiledRule, error) { |
| 454 | if m := reEq.FindStringSubmatch(expr); m != nil { |
| 455 | return compiledRule{kind: "eq", name: m[1], arg: m[2]}, nil |
| 456 | } |
| 457 | if m := reNe.FindStringSubmatch(expr); m != nil { |
| 458 | return compiledRule{kind: "ne", name: m[1], arg: m[2]}, nil |
| 459 | } |
| 460 | if m := reRe.FindStringSubmatch(expr); m != nil { |
| 461 | re, err := regexp.Compile(m[2]) |
| 462 | if err != nil { |
| 463 | return compiledRule{}, fmt.Errorf("ongeldige regex in rule %q: %w", expr, err) |
| 464 | } |
| 465 | return compiledRule{kind: "re", name: m[1], re: re}, nil |
| 466 | } |
| 467 | return compiledRule{}, fmt.Errorf("rule %q wordt niet begrepen", expr) |
| 468 | } |
| 469 | |
| 470 | func evalRule(expr string, env map[string]string) (bool, error) { |
| 471 | c, err := compileRule(expr) |
| 472 | if err != nil { |
| 473 | return false, err |
| 474 | } |
| 475 | val := env[c.name] |
| 476 | switch c.kind { |
| 477 | case "eq": |
| 478 | return val == c.arg, nil |
| 479 | case "ne": |
| 480 | return val != c.arg, nil |
| 481 | case "re": |
| 482 | return c.re.MatchString(val), nil |
| 483 | default: |
| 484 | return false, fmt.Errorf("interne rulefout") |
| 485 | } |
| 486 | } |
| 487 | |
| 488 | func parseExpire(s string) (time.Duration, error) { |
| 489 | s = strings.TrimSpace(s) |
| 490 | if s == "" || s == "never" { |
| 491 | return 0, nil |
| 492 | } |
| 493 | if len(s) < 2 { |
| 494 | return 0, fmt.Errorf("expire_in %q is ongeldig", s) |
| 495 | } |
| 496 | n, err := strconv.Atoi(s[:len(s)-1]) |
| 497 | if err != nil || n < 0 { |
| 498 | return 0, fmt.Errorf("expire_in %q is ongeldig", s) |
| 499 | } |
| 500 | switch s[len(s)-1] { |
| 501 | case 'm': |
| 502 | return time.Duration(n) * time.Minute, nil |
| 503 | case 'h': |
| 504 | return time.Duration(n) * time.Hour, nil |
| 505 | case 'd': |
| 506 | return time.Duration(n) * 24 * time.Hour, nil |
| 507 | case 'w': |
| 508 | return time.Duration(n) * 7 * 24 * time.Hour, nil |
| 509 | default: |
| 510 | return 0, fmt.Errorf("expire_in %q is ongeldig", s) |
| 511 | } |
| 512 | } |
| 513 | |
| 514 | func scalarList(node *yaml.Node) ([]string, error) { |
| 515 | if node.Kind != yaml.SequenceNode { |
| 516 | return nil, fmt.Errorf("verwacht een lijst") |
| 517 | } |
| 518 | var out []string |
| 519 | for _, item := range node.Content { |
| 520 | if item.Kind != yaml.ScalarNode { |
| 521 | return nil, fmt.Errorf("lijst mag alleen tekst bevatten") |
| 522 | } |
| 523 | out = append(out, item.Value) |
| 524 | } |
| 525 | return out, nil |
| 526 | } |
| 527 | |
| 528 | func scalarMap(node *yaml.Node) (map[string]string, error) { |
| 529 | if node.Kind != yaml.MappingNode { |
| 530 | return nil, fmt.Errorf("verwacht een mapping") |
| 531 | } |
| 532 | out := map[string]string{} |
| 533 | for i := 0; i < len(node.Content); i += 2 { |
| 534 | k := node.Content[i] |
| 535 | v := node.Content[i+1] |
| 536 | if k.Kind != yaml.ScalarNode || v.Kind != yaml.ScalarNode { |
| 537 | return nil, fmt.Errorf("variabelen moeten tekst zijn") |
| 538 | } |
| 539 | out[k.Value] = v.Value |
| 540 | } |
| 541 | return out, nil |
| 542 | } |
| 543 | |
| 544 | func parseBool(node *yaml.Node) (bool, error) { |
| 545 | if node.Kind != yaml.ScalarNode { |
| 546 | return false, fmt.Errorf("verwacht ja of nee") |
| 547 | } |
| 548 | switch strings.ToLower(node.Value) { |
| 549 | case "true", "yes", "ja": |
| 550 | return true, nil |
| 551 | case "false", "no", "nee": |
| 552 | return false, nil |
| 553 | default: |
| 554 | return false, fmt.Errorf("verwacht true of false") |
| 555 | } |
| 556 | } |
| 557 | |
| 558 | func safeRelPath(p string) error { |
| 559 | if p == "" || strings.HasPrefix(p, "/") || strings.Contains(p, "\\") { |
| 560 | return fmt.Errorf("pad %q moet relatief zijn", p) |
| 561 | } |
| 562 | for _, part := range strings.Split(p, "/") { |
| 563 | if part == ".." { |
| 564 | return fmt.Errorf("pad %q mag niet buiten de workspace", p) |
| 565 | } |
| 566 | } |
| 567 | return nil |
| 568 | } |