BlueGit

timfa bluegit

Files

bluegit / internal / ci / eval.go

eval.go

eval.go 15.8 KiB Download Raw
1package ci
2
3import (
4 "fmt"
5 "path"
6 "regexp"
7 "sort"
8 "strconv"
9 "strings"
10
11 "gitlab.com/Timfa/bluegit/internal/i18n"
12)
13
14// Variables zet de voorgedefinieerde CI_-variabelen. Job-specifieke namen
15// vult de runner zelf aan.
16func Variables(r Ref) map[string]string {
17 name := r.Name
18 kind := r.Kind
19 if kind == "" {
20 name, kind = SplitRef(r.FullRef)
21 }
22 short := r.SHA
23 if len(short) > 8 {
24 short = short[:8]
25 }
26 project := r.ProjectPath
27 ns, base := splitProject(project)
28 env := map[string]string{
29 "CI": "true",
30 "CI_CONFIG_PATH": "blue-ci-cd.yml",
31 "CI_SERVER_NAME": "BlueGit",
32 "CI_COMMIT_SHA": r.SHA,
33 "CI_COMMIT_SHORT_SHA": short,
34 "CI_COMMIT_REF": r.FullRef,
35 "CI_COMMIT_REF_NAME": name,
36 "CI_COMMIT_REF_SLUG": slug(name),
37 "CI_COMMIT_TITLE": r.Title,
38 "CI_COMMIT_BRANCH": "",
39 "CI_COMMIT_TAG": "",
40 "CI_DEFAULT_BRANCH": r.DefaultBranch,
41 "CI_PIPELINE_ID": fmt.Sprint(r.PipelineID),
42 "CI_PIPELINE_SOURCE": r.Source,
43 "CI_PIPELINE_URL": strings.TrimRight(r.ProjectURL, "/") + "/-/pipelines/" + fmt.Sprint(r.PipelineID),
44 "CI_PROJECT_PATH": project,
45 "CI_PROJECT_NAME": base,
46 "CI_PROJECT_NAMESPACE": ns,
47 "CI_PROJECT_URL": r.ProjectURL,
48 "CI_API_URL": r.APIURL,
49 }
50 if r.Timestamp != "" {
51 env["CI_COMMIT_TIMESTAMP"] = r.Timestamp
52 }
53 if kind == "tag" {
54 env["CI_COMMIT_TAG"] = name
55 } else {
56 env["CI_COMMIT_BRANCH"] = name
57 }
58 if env["CI_PIPELINE_SOURCE"] == "" {
59 env["CI_PIPELINE_SOURCE"] = "push"
60 }
61 if r.MR != nil {
62 m := r.MR
63 env["CI_MERGE_REQUEST_ID"] = fmt.Sprint(m.ID)
64 env["CI_MERGE_REQUEST_IID"] = fmt.Sprint(m.IID)
65 env["CI_MERGE_REQUEST_TITLE"] = m.Title
66 env["CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"] = m.SourceBranch
67 env["CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"] = r.SHA
68 env["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"] = m.TargetBranch
69 env["CI_MERGE_REQUEST_TARGET_BRANCH_SHA"] = m.TargetSHA
70 env["CI_MERGE_REQUEST_PROJECT_PATH"] = m.ProjectPath
71 env["CI_MERGE_REQUEST_PROJECT_URL"] = m.ProjectURL
72 }
73 return env
74}
75
76// SplitRef haalt de korte naam en het soort uit refs/heads of refs/tags.
77func SplitRef(ref string) (name, kind string) {
78 switch {
79 case strings.HasPrefix(ref, "refs/tags/"):
80 return strings.TrimPrefix(ref, "refs/tags/"), "tag"
81 case strings.HasPrefix(ref, "refs/heads/"):
82 return strings.TrimPrefix(ref, "refs/heads/"), "branch"
83 default:
84 return ref, "branch"
85 }
86}
87
88func splitProject(path string) (ns, name string) {
89 i := strings.LastIndex(path, "/")
90 if i < 0 {
91 return "", path
92 }
93 return path[:i], path[i+1:]
94}
95
96func slug(s string) string {
97 var b strings.Builder
98 prev := false
99 for _, r := range strings.ToLower(s) {
100 if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
101 b.WriteRune(r)
102 prev = false
103 continue
104 }
105 if !prev && b.Len() > 0 {
106 b.WriteByte('-')
107 prev = true
108 }
109 }
110 return strings.Trim(b.String(), "-")
111}
112
113// Select houdt de jobs waarvan de rules, only en except kloppen.
114// De When van de job wordt de when van de winnende rule.
115func (p Pipeline) Select(ev Eval) ([]Job, error) {
116 if ev.Env == nil {
117 ev.Env = map[string]string{}
118 }
119 if len(p.Workflow) > 0 && ev.Env["CI_PIPELINE_SOURCE"] == "merge_request_event" {
120 ev.workflowMR, _ = p.Allow(ev)
121 }
122 var out []Job
123 active := map[string]bool{}
124 var pending []Job
125 for _, j := range p.Jobs {
126 ok, when, err := j.matches(ev)
127 if err != nil {
128 return nil, fmt.Errorf("job %s: %w", j.Name, err)
129 }
130 if !ok {
131 continue
132 }
133 if when == "" {
134 when = "on_success"
135 }
136 j.When = when
137 pending = append(pending, j)
138 active[j.Name] = true
139 }
140 for i := range pending {
141 var needs []string
142 for _, n := range pending[i].Needs {
143 if active[n] {
144 needs = append(needs, n)
145 continue
146 }
147 if contains(pending[i].Optional, n) {
148 continue
149 }
150 needs = append(needs, n)
151 }
152 if needs == nil {
153 needs = []string{}
154 }
155 pending[i].Needs = needs
156 out = append(out, pending[i])
157 }
158 expanded, err := expandJobs(out)
159 if err != nil {
160 return nil, err
161 }
162 return p.byStage(expanded), nil
163}
164
165// byStage zet jobs in de volgorde van stages:, stabiel binnen een stage.
166// Gate en de runner leiden de stagevolgorde af uit de volgorde van de
167// slice. Een job die in het bestand vóór een eerdere stage staat, start
168// daardoor niet eerder.
169func (p Pipeline) byStage(jobs []Job) []Job {
170 if len(p.Stages) == 0 || len(jobs) < 2 {
171 return jobs
172 }
173 rank := map[string]int{}
174 for i, stage := range p.Stages {
175 rank[stage] = i
176 }
177 sort.SliceStable(jobs, func(i, j int) bool {
178 ri, okI := rank[jobs[i].Stage]
179 rj, okJ := rank[jobs[j].Stage]
180 if !okI {
181 ri = len(p.Stages)
182 }
183 if !okJ {
184 rj = len(p.Stages)
185 }
186 return ri < rj
187 })
188 return jobs
189}
190
191// Allow zegt of workflow:rules deze pipeline toelaat.
192// Zonder workflow wordt de pipeline gemaakt. Een passende rule met when:never
193// of geen enkele passende rule slaat de pipeline over.
194func (p Pipeline) Allow(ev Eval) (bool, error) {
195 if len(p.Workflow) == 0 {
196 return true, nil
197 }
198 if ev.Env == nil {
199 ev.Env = map[string]string{}
200 }
201 ok, _, err := (Job{Rules: p.Workflow, When: "always"}).matches(ev)
202 return ok, err
203}
204
205// ActiveJobs is Select met alleen variabelen. changes en exists gelden dan als waar.
206func (p Pipeline) ActiveJobs(env map[string]string) ([]Job, error) {
207 return p.Select(Eval{Env: env})
208}
209
210func (j Job) matches(ev Eval) (bool, string, error) {
211 when := j.When
212 if when == "" {
213 when = "on_success"
214 }
215 if len(j.Rules) == 0 {
216 ok, err := onlyExcept(j, ev)
217 return ok, when, err
218 }
219 for _, r := range j.Rules {
220 ok, err := ruleMatch(r, ev)
221 if err != nil {
222 return false, "", err
223 }
224 if !ok {
225 continue
226 }
227 w := r.When
228 if w == "" {
229 w = when
230 }
231 if w == "never" {
232 return false, w, nil
233 }
234 return true, w, nil
235 }
236 return false, "", nil
237}
238
239func ruleMatch(r Rule, ev Eval) (bool, error) {
240 if r.If != "" {
241 ok, err := evalRule(r.If, ev.Env)
242 if err != nil || !ok {
243 return ok, err
244 }
245 }
246 if len(r.Changes) > 0 && ev.Changed != nil {
247 if !anyGlob(r.Changes, ev.Changed) {
248 return false, nil
249 }
250 }
251 if len(r.Exists) > 0 && ev.Files != nil {
252 if !allExist(r.Exists, ev.Files) {
253 return false, nil
254 }
255 }
256 if r.If == "" && len(r.Changes) == 0 && len(r.Exists) == 0 && r.When == "" {
257 return false, nil
258 }
259 return true, nil
260}
261
262func onlyExcept(j Job, ev Eval) (bool, error) {
263 mr := ev.Env["CI_PIPELINE_SOURCE"] == "merge_request_event"
264 if len(j.Only) == 0 && len(j.Except) == 0 {
265 // Zonder rules of only draait een job op een branch of tag,
266 // niet in een merge-request-pipeline. Behalve als workflow:rules
267 // merge-request-pipelines expliciet toestaat: dan is een MR-pipeline
268 // zonder de gewone jobs nooit de bedoeling (anders ontstaat er geen).
269 if mr && !ev.workflowMR {
270 return false, nil
271 }
272 return true, nil
273 }
274 name := ev.Env["CI_COMMIT_REF_NAME"]
275 if name == "" {
276 name = ev.Env["CI_COMMIT_BRANCH"]
277 }
278 tag := ev.Env["CI_COMMIT_TAG"] != ""
279 if len(j.Only) > 0 {
280 ok, err := refListed(j.Only, name, tag, mr)
281 if err != nil || !ok {
282 return false, err
283 }
284 }
285 if len(j.Except) > 0 {
286 ok, err := refListed(j.Except, name, tag, mr)
287 if err != nil {
288 return false, err
289 }
290 if ok {
291 return false, nil
292 }
293 }
294 return true, nil
295}
296
297func refListed(list []string, name string, tag, mr bool) (bool, error) {
298 for _, item := range list {
299 switch item {
300 case "merge_requests":
301 if mr {
302 return true, nil
303 }
304 case "branches":
305 if !tag && !mr {
306 return true, nil
307 }
308 case "tags":
309 if tag && !mr {
310 return true, nil
311 }
312 default:
313 if mr {
314 continue
315 }
316 if strings.HasPrefix(item, "/") && strings.HasSuffix(item, "/") && len(item) > 2 {
317 re, err := regexp.Compile(item[1 : len(item)-1])
318 if err != nil {
319 return false, i18n.E("ci.err.regex", item, err.Error())
320 }
321 if re.MatchString(name) {
322 return true, nil
323 }
324 continue
325 }
326 if item == name {
327 return true, nil
328 }
329 }
330 }
331 return false, nil
332}
333
334func anyGlob(patterns, names []string) bool {
335 for _, name := range names {
336 for _, pattern := range patterns {
337 if Glob(pattern, name) {
338 return true
339 }
340 }
341 }
342 return false
343}
344
345func allExist(patterns, files []string) bool {
346 for _, pattern := range patterns {
347 found := false
348 for _, file := range files {
349 if file == pattern || Glob(pattern, file) {
350 found = true
351 break
352 }
353 }
354 if !found {
355 return false
356 }
357 }
358 return true
359}
360
361// Glob volgt GitLab-padpatronen: * blijft in één segment, ** loopt door mappen.
362func Glob(pattern, name string) bool {
363 pattern = strings.TrimPrefix(pattern, "/")
364 name = strings.TrimPrefix(name, "/")
365 if pattern == name {
366 return true
367 }
368 re := globRegexp(pattern)
369 if re.MatchString(name) {
370 return true
371 }
372 if !strings.Contains(pattern, "/") && re.MatchString(path.Base(name)) {
373 return true
374 }
375 return false
376}
377
378func globRegexp(pattern string) *regexp.Regexp {
379 var b strings.Builder
380 b.WriteString("^")
381 for i := 0; i < len(pattern); i++ {
382 if pattern[i] == '*' && i+1 < len(pattern) && pattern[i+1] == '*' {
383 b.WriteString(".*")
384 i++
385 if i+1 < len(pattern) && pattern[i+1] == '/' {
386 i++
387 }
388 continue
389 }
390 switch pattern[i] {
391 case '*':
392 b.WriteString("[^/]*")
393 case '?':
394 b.WriteString("[^/]")
395 default:
396 b.WriteString(regexp.QuoteMeta(pattern[i : i+1]))
397 }
398 }
399 b.WriteString("$")
400 return regexp.MustCompile(b.String())
401}
402
403// Gate beslist of een job mag starten.
404// "ready" betekent dat alle voorgangers klaar zijn.
405// "wait" betekent dat er nog een voorganger loopt of in de wachtrij staat.
406// "skip" betekent dat een voorganger mislukte zonder allow_failure,
407// dat een job in needs is overgeslagen, of dat when: on_failure geen
408// mislukking zag.
409// "fail" betekent dat een verplichte need niet in deze pipeline zit.
410//
411// NeedsSet volgt de genoemde jobs, ook als de lijst leeg is. Een
412// gefaalde of overgeslagen need slaat deze job over, behalve bij
413// allow_failure, when: on_failure en when: always. needs: [] heeft
414// geen voorgangers en start meteen.
415// Zonder needs wacht de job op alle jobs in eerdere stages. Een
416// overgeslagen eerdere stage telt dan als klaar; een harde mislukking
417// houdt latere stages tegen. De stagevolgorde is de volgorde waarin
418// de stages in de slice voor het eerst voorkomen. Select sorteert op
419// stages:, zodat een latere stage die bovenaan het bestand staat niet
420// eerder ready wordt.
421func Gate(jobs []Job, status map[string]string, job Job) string {
422 byName := map[string]Job{}
423 for _, j := range jobs {
424 byName[j.Name] = j
425 }
426 optional := map[string]bool{}
427 for _, n := range job.Optional {
428 optional[n] = true
429 }
430 var deps []Job
431 if job.NeedsSet {
432 for _, n := range job.Needs {
433 d, ok := byName[n]
434 if !ok {
435 if optional[n] {
436 continue
437 }
438 return "fail"
439 }
440 deps = append(deps, d)
441 }
442 } else {
443 stagePos := map[string]int{}
444 order := 0
445 for _, j := range jobs {
446 if _, ok := stagePos[j.Stage]; !ok {
447 stagePos[j.Stage] = order
448 order++
449 }
450 }
451 my := stagePos[job.Stage]
452 for _, j := range jobs {
453 if stagePos[j.Stage] < my {
454 deps = append(deps, j)
455 }
456 }
457 }
458 failedAny := false
459 failedHard := false
460 skippedNeed := false
461 for _, d := range deps {
462 switch status[d.Name] {
463 case "success":
464 continue
465 case "skipped":
466 // Alleen een genoemde need. Zonder needs telt een
467 // overgeslagen eerdere stage als afgerond.
468 if job.NeedsSet {
469 skippedNeed = true
470 }
471 case "failed", "canceled":
472 failedAny = true
473 if !d.AllowFailure {
474 failedHard = true
475 }
476 case "manual":
477 if d.AllowFailure {
478 continue
479 }
480 return "wait"
481 default:
482 return "wait"
483 }
484 }
485 when := job.When
486 if when == "" {
487 when = "on_success"
488 }
489 switch when {
490 case "always":
491 return "ready"
492 case "on_failure":
493 if failedAny {
494 return "ready"
495 }
496 return "skip"
497 default:
498 if failedHard || skippedNeed {
499 return "skip"
500 }
501 return "ready"
502 }
503}
504
505func evalRule(expr string, env map[string]string) (bool, error) {
506 p := &ruleParser{s: expr, env: env}
507 ok, err := p.parseOr()
508 if err != nil {
509 return false, err
510 }
511 p.skip()
512 if p.i < len(p.s) {
513 return false, i18n.E("ci.err.ruleExpr", strings.TrimSpace(expr))
514 }
515 return ok, nil
516}
517
518const maxParallel = 50
519
520func expandJobs(jobs []Job) ([]Job, error) {
521 expanded := map[string][]string{}
522 var out []Job
523 seen := map[string]bool{}
524 for _, j := range jobs {
525 instances, err := expandOne(j)
526 if err != nil {
527 return nil, fmt.Errorf("job %s: %w", j.Name, err)
528 }
529 names := make([]string, len(instances))
530 for i, inst := range instances {
531 if seen[inst.Name] {
532 return nil, i18n.E("ci.err.jobDup", inst.Name)
533 }
534 seen[inst.Name] = true
535 names[i] = inst.Name
536 out = append(out, inst)
537 }
538 expanded[j.Name] = names
539 }
540 for i := range out {
541 out[i].Needs = rewriteNeeds(out[i].Needs, expanded)
542 out[i].Optional = rewriteNeeds(out[i].Optional, expanded)
543 }
544 return out, nil
545}
546
547func expandOne(j Job) ([]Job, error) {
548 if len(j.Matrix) > 0 {
549 var combos []matrixCombo
550 for _, spec := range j.Matrix {
551 combos = append(combos, cartesian(spec)...)
552 }
553 if len(combos) > maxParallel {
554 return nil, i18n.E("ci.err.parallel", len(combos), maxParallel)
555 }
556 if len(combos) == 0 {
557 return nil, i18n.E("ci.err.parallel", 0, maxParallel)
558 }
559 var out []Job
560 for i, combo := range combos {
561 cp := copyJob(j)
562 cp.Name = j.Name + ": [" + strings.Join(combo.values, ", ") + "]"
563 cp.Parallel = 0
564 cp.Matrix = nil
565 for k, v := range combo.vars {
566 cp.Variables[k] = v
567 }
568 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i + 1)
569 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(len(combos))
570 out = append(out, cp)
571 }
572 return out, nil
573 }
574 if j.Parallel <= 1 {
575 return []Job{j}, nil
576 }
577 if j.Parallel > maxParallel {
578 return nil, i18n.E("ci.err.parallel", j.Parallel, maxParallel)
579 }
580 var out []Job
581 for i := 1; i <= j.Parallel; i++ {
582 cp := copyJob(j)
583 cp.Name = fmt.Sprintf("%s %d/%d", j.Name, i, j.Parallel)
584 cp.Parallel = 0
585 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i)
586 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(j.Parallel)
587 out = append(out, cp)
588 }
589 return out, nil
590}
591
592type matrixCombo struct {
593 values []string
594 vars map[string]string
595}
596
597func cartesian(spec MatrixSpec) []matrixCombo {
598 if len(spec.Keys) == 0 {
599 return nil
600 }
601 var out []matrixCombo
602 var walk func(i int, vals []string, vars map[string]string)
603 walk = func(i int, vals []string, vars map[string]string) {
604 if i == len(spec.Keys) {
605 cp := map[string]string{}
606 for k, v := range vars {
607 cp[k] = v
608 }
609 out = append(out, matrixCombo{values: append([]string{}, vals...), vars: cp})
610 return
611 }
612 for _, v := range spec.Vals[i] {
613 vars[spec.Keys[i]] = v
614 walk(i+1, append(vals, v), vars)
615 }
616 }
617 walk(0, nil, map[string]string{})
618 return out
619}
620
621func copyJob(j Job) Job {
622 cp := j
623 cp.Variables = map[string]string{}
624 for k, v := range j.Variables {
625 cp.Variables[k] = v
626 }
627 cp.Needs = append([]string{}, j.Needs...)
628 cp.Optional = append([]string{}, j.Optional...)
629 cp.Services = append([]Service{}, j.Services...)
630 return cp
631}
632
633func rewriteNeeds(list []string, expanded map[string][]string) []string {
634 if list == nil {
635 return nil
636 }
637 var out []string
638 for _, n := range list {
639 names := expanded[n]
640 if len(names) == 0 || (len(names) == 1 && names[0] == n) {
641 out = append(out, n)
642 continue
643 }
644 out = append(out, names...)
645 }
646 if out == nil {
647 return []string{}
648 }
649 return out
650}
651
652// Expand vervangt $NAAM en ${NAAM} in een tekst, bijvoorbeeld environment:url.
653func Expand(s string, env map[string]string) string {
654 var b strings.Builder
655 for i := 0; i < len(s); i++ {
656 if s[i] != '$' || i+1 >= len(s) {
657 b.WriteByte(s[i])
658 continue
659 }
660 if s[i+1] == '{' {
661 end := strings.IndexByte(s[i+2:], '}')
662 if end < 0 {
663 b.WriteByte('$')
664 continue
665 }
666 name := s[i+2 : i+2+end]
667 b.WriteString(env[name])
668 i += 2 + end
669 continue
670 }
671 j := i + 1
672 for j < len(s) && isIdent(s[j]) {
673 j++
674 }
675 if j == i+1 {
676 b.WriteByte('$')
677 continue
678 }
679 b.WriteString(env[s[i+1:j]])
680 i = j - 1
681 }
682 return b.String()
683}
684
685func contains(list []string, s string) bool {
686 for _, item := range list {
687 if item == s {
688 return true
689 }
690 }
691 return false
692}