BlueGit

timfa bluegit

Files

bluegit / internal / ci / eval.go

eval.go

fix: sla een job over als een need faalt of is overgeslagen
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
5d24c9de History
eval.go 14.5 KiB Download Raw
1package ci
2
3import (
4 "fmt"
5 "path"
6 "regexp"
7 "strconv"
8 "strings"
9
10 "gitlab.com/Timfa/bluegit/internal/i18n"
11)
12
13// Variables zet de voorgedefinieerde CI_-variabelen. Job-specifieke namen
14// vult de runner zelf aan.
15func Variables(r Ref) map[string]string {
16 name := r.Name
17 kind := r.Kind
18 if kind == "" {
19 name, kind = SplitRef(r.FullRef)
20 }
21 short := r.SHA
22 if len(short) > 8 {
23 short = short[:8]
24 }
25 project := r.ProjectPath
26 ns, base := splitProject(project)
27 env := map[string]string{
28 "CI": "true",
29 "CI_CONFIG_PATH": "blue-ci-cd.yml",
30 "CI_SERVER_NAME": "BlueGit",
31 "CI_COMMIT_SHA": r.SHA,
32 "CI_COMMIT_SHORT_SHA": short,
33 "CI_COMMIT_REF": r.FullRef,
34 "CI_COMMIT_REF_NAME": name,
35 "CI_COMMIT_REF_SLUG": slug(name),
36 "CI_COMMIT_TITLE": r.Title,
37 "CI_COMMIT_BRANCH": "",
38 "CI_COMMIT_TAG": "",
39 "CI_DEFAULT_BRANCH": r.DefaultBranch,
40 "CI_PIPELINE_ID": fmt.Sprint(r.PipelineID),
41 "CI_PIPELINE_SOURCE": r.Source,
42 "CI_PIPELINE_URL": strings.TrimRight(r.ProjectURL, "/") + "/-/pipelines/" + fmt.Sprint(r.PipelineID),
43 "CI_PROJECT_PATH": project,
44 "CI_PROJECT_NAME": base,
45 "CI_PROJECT_NAMESPACE": ns,
46 "CI_PROJECT_URL": r.ProjectURL,
47 "CI_API_URL": r.APIURL,
48 }
49 if kind == "tag" {
50 env["CI_COMMIT_TAG"] = name
51 } else {
52 env["CI_COMMIT_BRANCH"] = name
53 }
54 if env["CI_PIPELINE_SOURCE"] == "" {
55 env["CI_PIPELINE_SOURCE"] = "push"
56 }
57 if r.MR != nil {
58 m := r.MR
59 env["CI_MERGE_REQUEST_ID"] = fmt.Sprint(m.ID)
60 env["CI_MERGE_REQUEST_IID"] = fmt.Sprint(m.IID)
61 env["CI_MERGE_REQUEST_TITLE"] = m.Title
62 env["CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"] = m.SourceBranch
63 env["CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"] = r.SHA
64 env["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"] = m.TargetBranch
65 env["CI_MERGE_REQUEST_TARGET_BRANCH_SHA"] = m.TargetSHA
66 env["CI_MERGE_REQUEST_PROJECT_PATH"] = m.ProjectPath
67 env["CI_MERGE_REQUEST_PROJECT_URL"] = m.ProjectURL
68 }
69 return env
70}
71
72// SplitRef haalt de korte naam en het soort uit refs/heads of refs/tags.
73func SplitRef(ref string) (name, kind string) {
74 switch {
75 case strings.HasPrefix(ref, "refs/tags/"):
76 return strings.TrimPrefix(ref, "refs/tags/"), "tag"
77 case strings.HasPrefix(ref, "refs/heads/"):
78 return strings.TrimPrefix(ref, "refs/heads/"), "branch"
79 default:
80 return ref, "branch"
81 }
82}
83
84func splitProject(path string) (ns, name string) {
85 i := strings.LastIndex(path, "/")
86 if i < 0 {
87 return "", path
88 }
89 return path[:i], path[i+1:]
90}
91
92func slug(s string) string {
93 var b strings.Builder
94 prev := false
95 for _, r := range strings.ToLower(s) {
96 if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
97 b.WriteRune(r)
98 prev = false
99 continue
100 }
101 if !prev && b.Len() > 0 {
102 b.WriteByte('-')
103 prev = true
104 }
105 }
106 return strings.Trim(b.String(), "-")
107}
108
109// Select houdt de jobs waarvan de rules, only en except kloppen.
110// De When van de job wordt de when van de winnende rule.
111func (p Pipeline) Select(ev Eval) ([]Job, error) {
112 if ev.Env == nil {
113 ev.Env = map[string]string{}
114 }
115 var out []Job
116 active := map[string]bool{}
117 var pending []Job
118 for _, j := range p.Jobs {
119 ok, when, err := j.matches(ev)
120 if err != nil {
121 return nil, fmt.Errorf("job %s: %w", j.Name, err)
122 }
123 if !ok {
124 continue
125 }
126 if when == "" {
127 when = "on_success"
128 }
129 j.When = when
130 pending = append(pending, j)
131 active[j.Name] = true
132 }
133 for i := range pending {
134 var needs []string
135 for _, n := range pending[i].Needs {
136 if active[n] {
137 needs = append(needs, n)
138 continue
139 }
140 if contains(pending[i].Optional, n) {
141 continue
142 }
143 needs = append(needs, n)
144 }
145 if needs == nil {
146 needs = []string{}
147 }
148 pending[i].Needs = needs
149 out = append(out, pending[i])
150 }
151 return expandJobs(out)
152}
153
154// Allow zegt of workflow:rules deze pipeline toelaat.
155// Zonder workflow wordt de pipeline gemaakt. Een passende rule met when:never
156// of geen enkele passende rule slaat de pipeline over.
157func (p Pipeline) Allow(ev Eval) (bool, error) {
158 if len(p.Workflow) == 0 {
159 return true, nil
160 }
161 if ev.Env == nil {
162 ev.Env = map[string]string{}
163 }
164 ok, _, err := (Job{Rules: p.Workflow, When: "always"}).matches(ev)
165 return ok, err
166}
167
168// ActiveJobs is Select met alleen variabelen. changes en exists gelden dan als waar.
169func (p Pipeline) ActiveJobs(env map[string]string) ([]Job, error) {
170 return p.Select(Eval{Env: env})
171}
172
173func (j Job) matches(ev Eval) (bool, string, error) {
174 when := j.When
175 if when == "" {
176 when = "on_success"
177 }
178 if len(j.Rules) == 0 {
179 ok, err := onlyExcept(j, ev)
180 return ok, when, err
181 }
182 for _, r := range j.Rules {
183 ok, err := ruleMatch(r, ev)
184 if err != nil {
185 return false, "", err
186 }
187 if !ok {
188 continue
189 }
190 w := r.When
191 if w == "" {
192 w = when
193 }
194 if w == "never" {
195 return false, w, nil
196 }
197 return true, w, nil
198 }
199 return false, "", nil
200}
201
202func ruleMatch(r Rule, ev Eval) (bool, error) {
203 if r.If != "" {
204 ok, err := evalRule(r.If, ev.Env)
205 if err != nil || !ok {
206 return ok, err
207 }
208 }
209 if len(r.Changes) > 0 && ev.Changed != nil {
210 if !anyGlob(r.Changes, ev.Changed) {
211 return false, nil
212 }
213 }
214 if len(r.Exists) > 0 && ev.Files != nil {
215 if !allExist(r.Exists, ev.Files) {
216 return false, nil
217 }
218 }
219 if r.If == "" && len(r.Changes) == 0 && len(r.Exists) == 0 && r.When == "" {
220 return false, nil
221 }
222 return true, nil
223}
224
225func onlyExcept(j Job, ev Eval) (bool, error) {
226 mr := ev.Env["CI_PIPELINE_SOURCE"] == "merge_request_event"
227 if len(j.Only) == 0 && len(j.Except) == 0 {
228 // Zonder rules of only draait een job op een branch of tag,
229 // niet in een merge-request-pipeline.
230 if mr {
231 return false, nil
232 }
233 return true, nil
234 }
235 name := ev.Env["CI_COMMIT_REF_NAME"]
236 if name == "" {
237 name = ev.Env["CI_COMMIT_BRANCH"]
238 }
239 tag := ev.Env["CI_COMMIT_TAG"] != ""
240 if len(j.Only) > 0 {
241 ok, err := refListed(j.Only, name, tag, mr)
242 if err != nil || !ok {
243 return false, err
244 }
245 }
246 if len(j.Except) > 0 {
247 ok, err := refListed(j.Except, name, tag, mr)
248 if err != nil {
249 return false, err
250 }
251 if ok {
252 return false, nil
253 }
254 }
255 return true, nil
256}
257
258func refListed(list []string, name string, tag, mr bool) (bool, error) {
259 for _, item := range list {
260 switch item {
261 case "merge_requests":
262 if mr {
263 return true, nil
264 }
265 case "branches":
266 if !tag && !mr {
267 return true, nil
268 }
269 case "tags":
270 if tag && !mr {
271 return true, nil
272 }
273 default:
274 if mr {
275 continue
276 }
277 if strings.HasPrefix(item, "/") && strings.HasSuffix(item, "/") && len(item) > 2 {
278 re, err := regexp.Compile(item[1 : len(item)-1])
279 if err != nil {
280 return false, i18n.E("ci.err.regex", item, err.Error())
281 }
282 if re.MatchString(name) {
283 return true, nil
284 }
285 continue
286 }
287 if item == name {
288 return true, nil
289 }
290 }
291 }
292 return false, nil
293}
294
295func anyGlob(patterns, names []string) bool {
296 for _, name := range names {
297 for _, pattern := range patterns {
298 if Glob(pattern, name) {
299 return true
300 }
301 }
302 }
303 return false
304}
305
306func allExist(patterns, files []string) bool {
307 for _, pattern := range patterns {
308 found := false
309 for _, file := range files {
310 if file == pattern || Glob(pattern, file) {
311 found = true
312 break
313 }
314 }
315 if !found {
316 return false
317 }
318 }
319 return true
320}
321
322// Glob volgt GitLab-padpatronen: * blijft in één segment, ** loopt door mappen.
323func Glob(pattern, name string) bool {
324 pattern = strings.TrimPrefix(pattern, "/")
325 name = strings.TrimPrefix(name, "/")
326 if pattern == name {
327 return true
328 }
329 re := globRegexp(pattern)
330 if re.MatchString(name) {
331 return true
332 }
333 if !strings.Contains(pattern, "/") && re.MatchString(path.Base(name)) {
334 return true
335 }
336 return false
337}
338
339func globRegexp(pattern string) *regexp.Regexp {
340 var b strings.Builder
341 b.WriteString("^")
342 for i := 0; i < len(pattern); i++ {
343 if pattern[i] == '*' && i+1 < len(pattern) && pattern[i+1] == '*' {
344 b.WriteString(".*")
345 i++
346 if i+1 < len(pattern) && pattern[i+1] == '/' {
347 i++
348 }
349 continue
350 }
351 switch pattern[i] {
352 case '*':
353 b.WriteString("[^/]*")
354 case '?':
355 b.WriteString("[^/]")
356 default:
357 b.WriteString(regexp.QuoteMeta(pattern[i : i+1]))
358 }
359 }
360 b.WriteString("$")
361 return regexp.MustCompile(b.String())
362}
363
364// Gate beslist of een job mag starten.
365// "ready" betekent dat alle voorgangers klaar zijn.
366// "wait" betekent dat er nog een voorganger loopt of in de wachtrij staat.
367// "skip" betekent dat een voorganger mislukte zonder allow_failure,
368// dat een job in needs is overgeslagen, of dat when: on_failure geen
369// mislukking zag.
370// "fail" betekent dat een verplichte need niet in deze pipeline zit.
371//
372// NeedsSet volgt de genoemde jobs, ook als de lijst leeg is. Een
373// gefaalde of overgeslagen need slaat deze job over, behalve bij
374// allow_failure, when: on_failure en when: always. needs: [] heeft
375// geen voorgangers en start meteen.
376// Zonder needs wacht de job op alle jobs in eerdere stages. Een
377// overgeslagen eerdere stage telt dan als klaar; een harde mislukking
378// houdt latere stages tegen.
379func Gate(jobs []Job, status map[string]string, job Job) string {
380 byName := map[string]Job{}
381 for _, j := range jobs {
382 byName[j.Name] = j
383 }
384 optional := map[string]bool{}
385 for _, n := range job.Optional {
386 optional[n] = true
387 }
388 var deps []Job
389 if job.NeedsSet {
390 for _, n := range job.Needs {
391 d, ok := byName[n]
392 if !ok {
393 if optional[n] {
394 continue
395 }
396 return "fail"
397 }
398 deps = append(deps, d)
399 }
400 } else {
401 stagePos := map[string]int{}
402 order := 0
403 for _, j := range jobs {
404 if _, ok := stagePos[j.Stage]; !ok {
405 stagePos[j.Stage] = order
406 order++
407 }
408 }
409 my := stagePos[job.Stage]
410 for _, j := range jobs {
411 if stagePos[j.Stage] < my {
412 deps = append(deps, j)
413 }
414 }
415 }
416 failedAny := false
417 failedHard := false
418 skippedNeed := false
419 for _, d := range deps {
420 switch status[d.Name] {
421 case "success":
422 continue
423 case "skipped":
424 // Alleen een genoemde need. Zonder needs telt een
425 // overgeslagen eerdere stage als afgerond.
426 if job.NeedsSet {
427 skippedNeed = true
428 }
429 case "failed", "canceled":
430 failedAny = true
431 if !d.AllowFailure {
432 failedHard = true
433 }
434 case "manual":
435 if d.AllowFailure {
436 continue
437 }
438 return "wait"
439 default:
440 return "wait"
441 }
442 }
443 when := job.When
444 if when == "" {
445 when = "on_success"
446 }
447 switch when {
448 case "always":
449 return "ready"
450 case "on_failure":
451 if failedAny {
452 return "ready"
453 }
454 return "skip"
455 default:
456 if failedHard || skippedNeed {
457 return "skip"
458 }
459 return "ready"
460 }
461}
462
463func evalRule(expr string, env map[string]string) (bool, error) {
464 p := &ruleParser{s: expr, env: env}
465 ok, err := p.parseOr()
466 if err != nil {
467 return false, err
468 }
469 p.skip()
470 if p.i < len(p.s) {
471 return false, i18n.E("ci.err.ruleExpr", strings.TrimSpace(expr))
472 }
473 return ok, nil
474}
475
476const maxParallel = 50
477
478func expandJobs(jobs []Job) ([]Job, error) {
479 expanded := map[string][]string{}
480 var out []Job
481 seen := map[string]bool{}
482 for _, j := range jobs {
483 instances, err := expandOne(j)
484 if err != nil {
485 return nil, fmt.Errorf("job %s: %w", j.Name, err)
486 }
487 names := make([]string, len(instances))
488 for i, inst := range instances {
489 if seen[inst.Name] {
490 return nil, i18n.E("ci.err.jobDup", inst.Name)
491 }
492 seen[inst.Name] = true
493 names[i] = inst.Name
494 out = append(out, inst)
495 }
496 expanded[j.Name] = names
497 }
498 for i := range out {
499 out[i].Needs = rewriteNeeds(out[i].Needs, expanded)
500 out[i].Optional = rewriteNeeds(out[i].Optional, expanded)
501 }
502 return out, nil
503}
504
505func expandOne(j Job) ([]Job, error) {
506 if len(j.Matrix) > 0 {
507 var combos []matrixCombo
508 for _, spec := range j.Matrix {
509 combos = append(combos, cartesian(spec)...)
510 }
511 if len(combos) > maxParallel {
512 return nil, i18n.E("ci.err.parallel", len(combos), maxParallel)
513 }
514 if len(combos) == 0 {
515 return nil, i18n.E("ci.err.parallel", 0, maxParallel)
516 }
517 var out []Job
518 for i, combo := range combos {
519 cp := copyJob(j)
520 cp.Name = j.Name + ": [" + strings.Join(combo.values, ", ") + "]"
521 cp.Parallel = 0
522 cp.Matrix = nil
523 for k, v := range combo.vars {
524 cp.Variables[k] = v
525 }
526 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i + 1)
527 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(len(combos))
528 out = append(out, cp)
529 }
530 return out, nil
531 }
532 if j.Parallel <= 1 {
533 return []Job{j}, nil
534 }
535 if j.Parallel > maxParallel {
536 return nil, i18n.E("ci.err.parallel", j.Parallel, maxParallel)
537 }
538 var out []Job
539 for i := 1; i <= j.Parallel; i++ {
540 cp := copyJob(j)
541 cp.Name = fmt.Sprintf("%s %d/%d", j.Name, i, j.Parallel)
542 cp.Parallel = 0
543 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i)
544 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(j.Parallel)
545 out = append(out, cp)
546 }
547 return out, nil
548}
549
550type matrixCombo struct {
551 values []string
552 vars map[string]string
553}
554
555func cartesian(spec MatrixSpec) []matrixCombo {
556 if len(spec.Keys) == 0 {
557 return nil
558 }
559 var out []matrixCombo
560 var walk func(i int, vals []string, vars map[string]string)
561 walk = func(i int, vals []string, vars map[string]string) {
562 if i == len(spec.Keys) {
563 cp := map[string]string{}
564 for k, v := range vars {
565 cp[k] = v
566 }
567 out = append(out, matrixCombo{values: append([]string{}, vals...), vars: cp})
568 return
569 }
570 for _, v := range spec.Vals[i] {
571 vars[spec.Keys[i]] = v
572 walk(i+1, append(vals, v), vars)
573 }
574 }
575 walk(0, nil, map[string]string{})
576 return out
577}
578
579func copyJob(j Job) Job {
580 cp := j
581 cp.Variables = map[string]string{}
582 for k, v := range j.Variables {
583 cp.Variables[k] = v
584 }
585 cp.Needs = append([]string{}, j.Needs...)
586 cp.Optional = append([]string{}, j.Optional...)
587 cp.Services = append([]Service{}, j.Services...)
588 return cp
589}
590
591func rewriteNeeds(list []string, expanded map[string][]string) []string {
592 if list == nil {
593 return nil
594 }
595 var out []string
596 for _, n := range list {
597 names := expanded[n]
598 if len(names) == 0 || (len(names) == 1 && names[0] == n) {
599 out = append(out, n)
600 continue
601 }
602 out = append(out, names...)
603 }
604 if out == nil {
605 return []string{}
606 }
607 return out
608}
609
610// Expand vervangt $NAAM en ${NAAM} in een tekst, bijvoorbeeld environment:url.
611func Expand(s string, env map[string]string) string {
612 var b strings.Builder
613 for i := 0; i < len(s); i++ {
614 if s[i] != '$' || i+1 >= len(s) {
615 b.WriteByte(s[i])
616 continue
617 }
618 if s[i+1] == '{' {
619 end := strings.IndexByte(s[i+2:], '}')
620 if end < 0 {
621 b.WriteByte('$')
622 continue
623 }
624 name := s[i+2 : i+2+end]
625 b.WriteString(env[name])
626 i += 2 + end
627 continue
628 }
629 j := i + 1
630 for j < len(s) && isIdent(s[j]) {
631 j++
632 }
633 if j == i+1 {
634 b.WriteByte('$')
635 continue
636 }
637 b.WriteString(env[s[i+1:j]])
638 i = j - 1
639 }
640 return b.String()
641}
642
643func contains(list []string, s string) bool {
644 for _, item := range list {
645 if item == s {
646 return true
647 }
648 }
649 return false
650}