BlueGit

timfa bluegit

Files

bluegit / internal / ci / eval.go

eval.go

fix: laat go-tests in Steve slagen zonder namespaces of smart HTTP
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
4bb9fce0 History
eval.go 15.5 KiB Download Raw
1package ci
2
3import (
4 "fmt"
5 "path"
6 "regexp"
7 "sort"
8 "strconv"
9 "strings"
10
11 "gitlab.com/Timfa/bluegit/internal/i18n"
12)
13
14// Variables zet de voorgedefinieerde CI_-variabelen. Job-specifieke namen
15// vult de runner zelf aan.
16func Variables(r Ref) map[string]string {
17 name := r.Name
18 kind := r.Kind
19 if kind == "" {
20 name, kind = SplitRef(r.FullRef)
21 }
22 short := r.SHA
23 if len(short) > 8 {
24 short = short[:8]
25 }
26 project := r.ProjectPath
27 ns, base := splitProject(project)
28 env := map[string]string{
29 "CI": "true",
30 "CI_CONFIG_PATH": "blue-ci-cd.yml",
31 "CI_SERVER_NAME": "BlueGit",
32 "CI_COMMIT_SHA": r.SHA,
33 "CI_COMMIT_SHORT_SHA": short,
34 "CI_COMMIT_REF": r.FullRef,
35 "CI_COMMIT_REF_NAME": name,
36 "CI_COMMIT_REF_SLUG": slug(name),
37 "CI_COMMIT_TITLE": r.Title,
38 "CI_COMMIT_BRANCH": "",
39 "CI_COMMIT_TAG": "",
40 "CI_DEFAULT_BRANCH": r.DefaultBranch,
41 "CI_PIPELINE_ID": fmt.Sprint(r.PipelineID),
42 "CI_PIPELINE_SOURCE": r.Source,
43 "CI_PIPELINE_URL": strings.TrimRight(r.ProjectURL, "/") + "/-/pipelines/" + fmt.Sprint(r.PipelineID),
44 "CI_PROJECT_PATH": project,
45 "CI_PROJECT_NAME": base,
46 "CI_PROJECT_NAMESPACE": ns,
47 "CI_PROJECT_URL": r.ProjectURL,
48 "CI_API_URL": r.APIURL,
49 }
50 if r.Timestamp != "" {
51 env["CI_COMMIT_TIMESTAMP"] = r.Timestamp
52 }
53 if kind == "tag" {
54 env["CI_COMMIT_TAG"] = name
55 } else {
56 env["CI_COMMIT_BRANCH"] = name
57 }
58 if env["CI_PIPELINE_SOURCE"] == "" {
59 env["CI_PIPELINE_SOURCE"] = "push"
60 }
61 if r.MR != nil {
62 m := r.MR
63 env["CI_MERGE_REQUEST_ID"] = fmt.Sprint(m.ID)
64 env["CI_MERGE_REQUEST_IID"] = fmt.Sprint(m.IID)
65 env["CI_MERGE_REQUEST_TITLE"] = m.Title
66 env["CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"] = m.SourceBranch
67 env["CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"] = r.SHA
68 env["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"] = m.TargetBranch
69 env["CI_MERGE_REQUEST_TARGET_BRANCH_SHA"] = m.TargetSHA
70 env["CI_MERGE_REQUEST_PROJECT_PATH"] = m.ProjectPath
71 env["CI_MERGE_REQUEST_PROJECT_URL"] = m.ProjectURL
72 }
73 return env
74}
75
76// SplitRef haalt de korte naam en het soort uit refs/heads of refs/tags.
77func SplitRef(ref string) (name, kind string) {
78 switch {
79 case strings.HasPrefix(ref, "refs/tags/"):
80 return strings.TrimPrefix(ref, "refs/tags/"), "tag"
81 case strings.HasPrefix(ref, "refs/heads/"):
82 return strings.TrimPrefix(ref, "refs/heads/"), "branch"
83 default:
84 return ref, "branch"
85 }
86}
87
88func splitProject(path string) (ns, name string) {
89 i := strings.LastIndex(path, "/")
90 if i < 0 {
91 return "", path
92 }
93 return path[:i], path[i+1:]
94}
95
96func slug(s string) string {
97 var b strings.Builder
98 prev := false
99 for _, r := range strings.ToLower(s) {
100 if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
101 b.WriteRune(r)
102 prev = false
103 continue
104 }
105 if !prev && b.Len() > 0 {
106 b.WriteByte('-')
107 prev = true
108 }
109 }
110 return strings.Trim(b.String(), "-")
111}
112
113// Select houdt de jobs waarvan de rules, only en except kloppen.
114// De When van de job wordt de when van de winnende rule.
115func (p Pipeline) Select(ev Eval) ([]Job, error) {
116 if ev.Env == nil {
117 ev.Env = map[string]string{}
118 }
119 var out []Job
120 active := map[string]bool{}
121 var pending []Job
122 for _, j := range p.Jobs {
123 ok, when, err := j.matches(ev)
124 if err != nil {
125 return nil, fmt.Errorf("job %s: %w", j.Name, err)
126 }
127 if !ok {
128 continue
129 }
130 if when == "" {
131 when = "on_success"
132 }
133 j.When = when
134 pending = append(pending, j)
135 active[j.Name] = true
136 }
137 for i := range pending {
138 var needs []string
139 for _, n := range pending[i].Needs {
140 if active[n] {
141 needs = append(needs, n)
142 continue
143 }
144 if contains(pending[i].Optional, n) {
145 continue
146 }
147 needs = append(needs, n)
148 }
149 if needs == nil {
150 needs = []string{}
151 }
152 pending[i].Needs = needs
153 out = append(out, pending[i])
154 }
155 expanded, err := expandJobs(out)
156 if err != nil {
157 return nil, err
158 }
159 return p.byStage(expanded), nil
160}
161
162// byStage zet jobs in de volgorde van stages:, stabiel binnen een stage.
163// Gate en de runner leiden de stagevolgorde af uit de volgorde van de
164// slice. Een job die in het bestand vóór een eerdere stage staat, start
165// daardoor niet eerder.
166func (p Pipeline) byStage(jobs []Job) []Job {
167 if len(p.Stages) == 0 || len(jobs) < 2 {
168 return jobs
169 }
170 rank := map[string]int{}
171 for i, stage := range p.Stages {
172 rank[stage] = i
173 }
174 sort.SliceStable(jobs, func(i, j int) bool {
175 ri, okI := rank[jobs[i].Stage]
176 rj, okJ := rank[jobs[j].Stage]
177 if !okI {
178 ri = len(p.Stages)
179 }
180 if !okJ {
181 rj = len(p.Stages)
182 }
183 return ri < rj
184 })
185 return jobs
186}
187
188// Allow zegt of workflow:rules deze pipeline toelaat.
189// Zonder workflow wordt de pipeline gemaakt. Een passende rule met when:never
190// of geen enkele passende rule slaat de pipeline over.
191func (p Pipeline) Allow(ev Eval) (bool, error) {
192 if len(p.Workflow) == 0 {
193 return true, nil
194 }
195 if ev.Env == nil {
196 ev.Env = map[string]string{}
197 }
198 ok, _, err := (Job{Rules: p.Workflow, When: "always"}).matches(ev)
199 return ok, err
200}
201
202// ActiveJobs is Select met alleen variabelen. changes en exists gelden dan als waar.
203func (p Pipeline) ActiveJobs(env map[string]string) ([]Job, error) {
204 return p.Select(Eval{Env: env})
205}
206
207func (j Job) matches(ev Eval) (bool, string, error) {
208 when := j.When
209 if when == "" {
210 when = "on_success"
211 }
212 if len(j.Rules) == 0 {
213 ok, err := onlyExcept(j, ev)
214 return ok, when, err
215 }
216 for _, r := range j.Rules {
217 ok, err := ruleMatch(r, ev)
218 if err != nil {
219 return false, "", err
220 }
221 if !ok {
222 continue
223 }
224 w := r.When
225 if w == "" {
226 w = when
227 }
228 if w == "never" {
229 return false, w, nil
230 }
231 return true, w, nil
232 }
233 return false, "", nil
234}
235
236func ruleMatch(r Rule, ev Eval) (bool, error) {
237 if r.If != "" {
238 ok, err := evalRule(r.If, ev.Env)
239 if err != nil || !ok {
240 return ok, err
241 }
242 }
243 if len(r.Changes) > 0 && ev.Changed != nil {
244 if !anyGlob(r.Changes, ev.Changed) {
245 return false, nil
246 }
247 }
248 if len(r.Exists) > 0 && ev.Files != nil {
249 if !allExist(r.Exists, ev.Files) {
250 return false, nil
251 }
252 }
253 if r.If == "" && len(r.Changes) == 0 && len(r.Exists) == 0 && r.When == "" {
254 return false, nil
255 }
256 return true, nil
257}
258
259func onlyExcept(j Job, ev Eval) (bool, error) {
260 mr := ev.Env["CI_PIPELINE_SOURCE"] == "merge_request_event"
261 if len(j.Only) == 0 && len(j.Except) == 0 {
262 // Zonder rules of only draait een job op een branch of tag,
263 // niet in een merge-request-pipeline.
264 if mr {
265 return false, nil
266 }
267 return true, nil
268 }
269 name := ev.Env["CI_COMMIT_REF_NAME"]
270 if name == "" {
271 name = ev.Env["CI_COMMIT_BRANCH"]
272 }
273 tag := ev.Env["CI_COMMIT_TAG"] != ""
274 if len(j.Only) > 0 {
275 ok, err := refListed(j.Only, name, tag, mr)
276 if err != nil || !ok {
277 return false, err
278 }
279 }
280 if len(j.Except) > 0 {
281 ok, err := refListed(j.Except, name, tag, mr)
282 if err != nil {
283 return false, err
284 }
285 if ok {
286 return false, nil
287 }
288 }
289 return true, nil
290}
291
292func refListed(list []string, name string, tag, mr bool) (bool, error) {
293 for _, item := range list {
294 switch item {
295 case "merge_requests":
296 if mr {
297 return true, nil
298 }
299 case "branches":
300 if !tag && !mr {
301 return true, nil
302 }
303 case "tags":
304 if tag && !mr {
305 return true, nil
306 }
307 default:
308 if mr {
309 continue
310 }
311 if strings.HasPrefix(item, "/") && strings.HasSuffix(item, "/") && len(item) > 2 {
312 re, err := regexp.Compile(item[1 : len(item)-1])
313 if err != nil {
314 return false, i18n.E("ci.err.regex", item, err.Error())
315 }
316 if re.MatchString(name) {
317 return true, nil
318 }
319 continue
320 }
321 if item == name {
322 return true, nil
323 }
324 }
325 }
326 return false, nil
327}
328
329func anyGlob(patterns, names []string) bool {
330 for _, name := range names {
331 for _, pattern := range patterns {
332 if Glob(pattern, name) {
333 return true
334 }
335 }
336 }
337 return false
338}
339
340func allExist(patterns, files []string) bool {
341 for _, pattern := range patterns {
342 found := false
343 for _, file := range files {
344 if file == pattern || Glob(pattern, file) {
345 found = true
346 break
347 }
348 }
349 if !found {
350 return false
351 }
352 }
353 return true
354}
355
356// Glob volgt GitLab-padpatronen: * blijft in één segment, ** loopt door mappen.
357func Glob(pattern, name string) bool {
358 pattern = strings.TrimPrefix(pattern, "/")
359 name = strings.TrimPrefix(name, "/")
360 if pattern == name {
361 return true
362 }
363 re := globRegexp(pattern)
364 if re.MatchString(name) {
365 return true
366 }
367 if !strings.Contains(pattern, "/") && re.MatchString(path.Base(name)) {
368 return true
369 }
370 return false
371}
372
373func globRegexp(pattern string) *regexp.Regexp {
374 var b strings.Builder
375 b.WriteString("^")
376 for i := 0; i < len(pattern); i++ {
377 if pattern[i] == '*' && i+1 < len(pattern) && pattern[i+1] == '*' {
378 b.WriteString(".*")
379 i++
380 if i+1 < len(pattern) && pattern[i+1] == '/' {
381 i++
382 }
383 continue
384 }
385 switch pattern[i] {
386 case '*':
387 b.WriteString("[^/]*")
388 case '?':
389 b.WriteString("[^/]")
390 default:
391 b.WriteString(regexp.QuoteMeta(pattern[i : i+1]))
392 }
393 }
394 b.WriteString("$")
395 return regexp.MustCompile(b.String())
396}
397
398// Gate beslist of een job mag starten.
399// "ready" betekent dat alle voorgangers klaar zijn.
400// "wait" betekent dat er nog een voorganger loopt of in de wachtrij staat.
401// "skip" betekent dat een voorganger mislukte zonder allow_failure,
402// dat een job in needs is overgeslagen, of dat when: on_failure geen
403// mislukking zag.
404// "fail" betekent dat een verplichte need niet in deze pipeline zit.
405//
406// NeedsSet volgt de genoemde jobs, ook als de lijst leeg is. Een
407// gefaalde of overgeslagen need slaat deze job over, behalve bij
408// allow_failure, when: on_failure en when: always. needs: [] heeft
409// geen voorgangers en start meteen.
410// Zonder needs wacht de job op alle jobs in eerdere stages. Een
411// overgeslagen eerdere stage telt dan als klaar; een harde mislukking
412// houdt latere stages tegen. De stagevolgorde is de volgorde waarin
413// de stages in de slice voor het eerst voorkomen. Select sorteert op
414// stages:, zodat een latere stage die bovenaan het bestand staat niet
415// eerder ready wordt.
416func Gate(jobs []Job, status map[string]string, job Job) string {
417 byName := map[string]Job{}
418 for _, j := range jobs {
419 byName[j.Name] = j
420 }
421 optional := map[string]bool{}
422 for _, n := range job.Optional {
423 optional[n] = true
424 }
425 var deps []Job
426 if job.NeedsSet {
427 for _, n := range job.Needs {
428 d, ok := byName[n]
429 if !ok {
430 if optional[n] {
431 continue
432 }
433 return "fail"
434 }
435 deps = append(deps, d)
436 }
437 } else {
438 stagePos := map[string]int{}
439 order := 0
440 for _, j := range jobs {
441 if _, ok := stagePos[j.Stage]; !ok {
442 stagePos[j.Stage] = order
443 order++
444 }
445 }
446 my := stagePos[job.Stage]
447 for _, j := range jobs {
448 if stagePos[j.Stage] < my {
449 deps = append(deps, j)
450 }
451 }
452 }
453 failedAny := false
454 failedHard := false
455 skippedNeed := false
456 for _, d := range deps {
457 switch status[d.Name] {
458 case "success":
459 continue
460 case "skipped":
461 // Alleen een genoemde need. Zonder needs telt een
462 // overgeslagen eerdere stage als afgerond.
463 if job.NeedsSet {
464 skippedNeed = true
465 }
466 case "failed", "canceled":
467 failedAny = true
468 if !d.AllowFailure {
469 failedHard = true
470 }
471 case "manual":
472 if d.AllowFailure {
473 continue
474 }
475 return "wait"
476 default:
477 return "wait"
478 }
479 }
480 when := job.When
481 if when == "" {
482 when = "on_success"
483 }
484 switch when {
485 case "always":
486 return "ready"
487 case "on_failure":
488 if failedAny {
489 return "ready"
490 }
491 return "skip"
492 default:
493 if failedHard || skippedNeed {
494 return "skip"
495 }
496 return "ready"
497 }
498}
499
500func evalRule(expr string, env map[string]string) (bool, error) {
501 p := &ruleParser{s: expr, env: env}
502 ok, err := p.parseOr()
503 if err != nil {
504 return false, err
505 }
506 p.skip()
507 if p.i < len(p.s) {
508 return false, i18n.E("ci.err.ruleExpr", strings.TrimSpace(expr))
509 }
510 return ok, nil
511}
512
513const maxParallel = 50
514
515func expandJobs(jobs []Job) ([]Job, error) {
516 expanded := map[string][]string{}
517 var out []Job
518 seen := map[string]bool{}
519 for _, j := range jobs {
520 instances, err := expandOne(j)
521 if err != nil {
522 return nil, fmt.Errorf("job %s: %w", j.Name, err)
523 }
524 names := make([]string, len(instances))
525 for i, inst := range instances {
526 if seen[inst.Name] {
527 return nil, i18n.E("ci.err.jobDup", inst.Name)
528 }
529 seen[inst.Name] = true
530 names[i] = inst.Name
531 out = append(out, inst)
532 }
533 expanded[j.Name] = names
534 }
535 for i := range out {
536 out[i].Needs = rewriteNeeds(out[i].Needs, expanded)
537 out[i].Optional = rewriteNeeds(out[i].Optional, expanded)
538 }
539 return out, nil
540}
541
542func expandOne(j Job) ([]Job, error) {
543 if len(j.Matrix) > 0 {
544 var combos []matrixCombo
545 for _, spec := range j.Matrix {
546 combos = append(combos, cartesian(spec)...)
547 }
548 if len(combos) > maxParallel {
549 return nil, i18n.E("ci.err.parallel", len(combos), maxParallel)
550 }
551 if len(combos) == 0 {
552 return nil, i18n.E("ci.err.parallel", 0, maxParallel)
553 }
554 var out []Job
555 for i, combo := range combos {
556 cp := copyJob(j)
557 cp.Name = j.Name + ": [" + strings.Join(combo.values, ", ") + "]"
558 cp.Parallel = 0
559 cp.Matrix = nil
560 for k, v := range combo.vars {
561 cp.Variables[k] = v
562 }
563 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i + 1)
564 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(len(combos))
565 out = append(out, cp)
566 }
567 return out, nil
568 }
569 if j.Parallel <= 1 {
570 return []Job{j}, nil
571 }
572 if j.Parallel > maxParallel {
573 return nil, i18n.E("ci.err.parallel", j.Parallel, maxParallel)
574 }
575 var out []Job
576 for i := 1; i <= j.Parallel; i++ {
577 cp := copyJob(j)
578 cp.Name = fmt.Sprintf("%s %d/%d", j.Name, i, j.Parallel)
579 cp.Parallel = 0
580 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i)
581 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(j.Parallel)
582 out = append(out, cp)
583 }
584 return out, nil
585}
586
587type matrixCombo struct {
588 values []string
589 vars map[string]string
590}
591
592func cartesian(spec MatrixSpec) []matrixCombo {
593 if len(spec.Keys) == 0 {
594 return nil
595 }
596 var out []matrixCombo
597 var walk func(i int, vals []string, vars map[string]string)
598 walk = func(i int, vals []string, vars map[string]string) {
599 if i == len(spec.Keys) {
600 cp := map[string]string{}
601 for k, v := range vars {
602 cp[k] = v
603 }
604 out = append(out, matrixCombo{values: append([]string{}, vals...), vars: cp})
605 return
606 }
607 for _, v := range spec.Vals[i] {
608 vars[spec.Keys[i]] = v
609 walk(i+1, append(vals, v), vars)
610 }
611 }
612 walk(0, nil, map[string]string{})
613 return out
614}
615
616func copyJob(j Job) Job {
617 cp := j
618 cp.Variables = map[string]string{}
619 for k, v := range j.Variables {
620 cp.Variables[k] = v
621 }
622 cp.Needs = append([]string{}, j.Needs...)
623 cp.Optional = append([]string{}, j.Optional...)
624 cp.Services = append([]Service{}, j.Services...)
625 return cp
626}
627
628func rewriteNeeds(list []string, expanded map[string][]string) []string {
629 if list == nil {
630 return nil
631 }
632 var out []string
633 for _, n := range list {
634 names := expanded[n]
635 if len(names) == 0 || (len(names) == 1 && names[0] == n) {
636 out = append(out, n)
637 continue
638 }
639 out = append(out, names...)
640 }
641 if out == nil {
642 return []string{}
643 }
644 return out
645}
646
647// Expand vervangt $NAAM en ${NAAM} in een tekst, bijvoorbeeld environment:url.
648func Expand(s string, env map[string]string) string {
649 var b strings.Builder
650 for i := 0; i < len(s); i++ {
651 if s[i] != '$' || i+1 >= len(s) {
652 b.WriteByte(s[i])
653 continue
654 }
655 if s[i+1] == '{' {
656 end := strings.IndexByte(s[i+2:], '}')
657 if end < 0 {
658 b.WriteByte('$')
659 continue
660 }
661 name := s[i+2 : i+2+end]
662 b.WriteString(env[name])
663 i += 2 + end
664 continue
665 }
666 j := i + 1
667 for j < len(s) && isIdent(s[j]) {
668 j++
669 }
670 if j == i+1 {
671 b.WriteByte('$')
672 continue
673 }
674 b.WriteString(env[s[i+1:j]])
675 i = j - 1
676 }
677 return b.String()
678}
679
680func contains(list []string, s string) bool {
681 for _, item := range list {
682 if item == s {
683 return true
684 }
685 }
686 return false
687}