BlueGit

timfa bluegit

Files

bluegit / internal / ci / eval.go

eval.go

feat: breid rules en de ingebouwde runner uit
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
5fec96f0 History
eval.go 14.0 KiB Download Raw
1package ci
2
3import (
4 "fmt"
5 "path"
6 "regexp"
7 "strconv"
8 "strings"
9
10 "gitlab.com/Timfa/bluegit/internal/i18n"
11)
12
13// Variables zet de voorgedefinieerde CI_-variabelen. Job-specifieke namen
14// vult de runner zelf aan.
15func Variables(r Ref) map[string]string {
16 name := r.Name
17 kind := r.Kind
18 if kind == "" {
19 name, kind = SplitRef(r.FullRef)
20 }
21 short := r.SHA
22 if len(short) > 8 {
23 short = short[:8]
24 }
25 project := r.ProjectPath
26 ns, base := splitProject(project)
27 env := map[string]string{
28 "CI": "true",
29 "CI_CONFIG_PATH": "blue-ci-cd.yml",
30 "CI_SERVER_NAME": "BlueGit",
31 "CI_COMMIT_SHA": r.SHA,
32 "CI_COMMIT_SHORT_SHA": short,
33 "CI_COMMIT_REF": r.FullRef,
34 "CI_COMMIT_REF_NAME": name,
35 "CI_COMMIT_REF_SLUG": slug(name),
36 "CI_COMMIT_TITLE": r.Title,
37 "CI_COMMIT_BRANCH": "",
38 "CI_COMMIT_TAG": "",
39 "CI_DEFAULT_BRANCH": r.DefaultBranch,
40 "CI_PIPELINE_ID": fmt.Sprint(r.PipelineID),
41 "CI_PIPELINE_SOURCE": r.Source,
42 "CI_PIPELINE_URL": strings.TrimRight(r.ProjectURL, "/") + "/-/pipelines/" + fmt.Sprint(r.PipelineID),
43 "CI_PROJECT_PATH": project,
44 "CI_PROJECT_NAME": base,
45 "CI_PROJECT_NAMESPACE": ns,
46 "CI_PROJECT_URL": r.ProjectURL,
47 "CI_API_URL": r.APIURL,
48 }
49 if kind == "tag" {
50 env["CI_COMMIT_TAG"] = name
51 } else {
52 env["CI_COMMIT_BRANCH"] = name
53 }
54 if env["CI_PIPELINE_SOURCE"] == "" {
55 env["CI_PIPELINE_SOURCE"] = "push"
56 }
57 if r.MR != nil {
58 m := r.MR
59 env["CI_MERGE_REQUEST_ID"] = fmt.Sprint(m.ID)
60 env["CI_MERGE_REQUEST_IID"] = fmt.Sprint(m.IID)
61 env["CI_MERGE_REQUEST_TITLE"] = m.Title
62 env["CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"] = m.SourceBranch
63 env["CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"] = r.SHA
64 env["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"] = m.TargetBranch
65 env["CI_MERGE_REQUEST_TARGET_BRANCH_SHA"] = m.TargetSHA
66 env["CI_MERGE_REQUEST_PROJECT_PATH"] = m.ProjectPath
67 env["CI_MERGE_REQUEST_PROJECT_URL"] = m.ProjectURL
68 }
69 return env
70}
71
72// SplitRef haalt de korte naam en het soort uit refs/heads of refs/tags.
73func SplitRef(ref string) (name, kind string) {
74 switch {
75 case strings.HasPrefix(ref, "refs/tags/"):
76 return strings.TrimPrefix(ref, "refs/tags/"), "tag"
77 case strings.HasPrefix(ref, "refs/heads/"):
78 return strings.TrimPrefix(ref, "refs/heads/"), "branch"
79 default:
80 return ref, "branch"
81 }
82}
83
84func splitProject(path string) (ns, name string) {
85 i := strings.LastIndex(path, "/")
86 if i < 0 {
87 return "", path
88 }
89 return path[:i], path[i+1:]
90}
91
92func slug(s string) string {
93 var b strings.Builder
94 prev := false
95 for _, r := range strings.ToLower(s) {
96 if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
97 b.WriteRune(r)
98 prev = false
99 continue
100 }
101 if !prev && b.Len() > 0 {
102 b.WriteByte('-')
103 prev = true
104 }
105 }
106 return strings.Trim(b.String(), "-")
107}
108
109// Select houdt de jobs waarvan de rules, only en except kloppen.
110// De When van de job wordt de when van de winnende rule.
111func (p Pipeline) Select(ev Eval) ([]Job, error) {
112 if ev.Env == nil {
113 ev.Env = map[string]string{}
114 }
115 var out []Job
116 active := map[string]bool{}
117 var pending []Job
118 for _, j := range p.Jobs {
119 ok, when, err := j.matches(ev)
120 if err != nil {
121 return nil, fmt.Errorf("job %s: %w", j.Name, err)
122 }
123 if !ok {
124 continue
125 }
126 if when == "" {
127 when = "on_success"
128 }
129 j.When = when
130 pending = append(pending, j)
131 active[j.Name] = true
132 }
133 for i := range pending {
134 var needs []string
135 for _, n := range pending[i].Needs {
136 if active[n] {
137 needs = append(needs, n)
138 continue
139 }
140 if contains(pending[i].Optional, n) {
141 continue
142 }
143 needs = append(needs, n)
144 }
145 if needs == nil {
146 needs = []string{}
147 }
148 pending[i].Needs = needs
149 out = append(out, pending[i])
150 }
151 return expandJobs(out)
152}
153
154// Allow zegt of workflow:rules deze pipeline toelaat.
155// Zonder workflow wordt de pipeline gemaakt. Een passende rule met when:never
156// of geen enkele passende rule slaat de pipeline over.
157func (p Pipeline) Allow(ev Eval) (bool, error) {
158 if len(p.Workflow) == 0 {
159 return true, nil
160 }
161 if ev.Env == nil {
162 ev.Env = map[string]string{}
163 }
164 ok, _, err := (Job{Rules: p.Workflow, When: "always"}).matches(ev)
165 return ok, err
166}
167
168// ActiveJobs is Select met alleen variabelen. changes en exists gelden dan als waar.
169func (p Pipeline) ActiveJobs(env map[string]string) ([]Job, error) {
170 return p.Select(Eval{Env: env})
171}
172
173func (j Job) matches(ev Eval) (bool, string, error) {
174 when := j.When
175 if when == "" {
176 when = "on_success"
177 }
178 if len(j.Rules) == 0 {
179 ok, err := onlyExcept(j, ev)
180 return ok, when, err
181 }
182 for _, r := range j.Rules {
183 ok, err := ruleMatch(r, ev)
184 if err != nil {
185 return false, "", err
186 }
187 if !ok {
188 continue
189 }
190 w := r.When
191 if w == "" {
192 w = when
193 }
194 if w == "never" {
195 return false, w, nil
196 }
197 return true, w, nil
198 }
199 return false, "", nil
200}
201
202func ruleMatch(r Rule, ev Eval) (bool, error) {
203 if r.If != "" {
204 ok, err := evalRule(r.If, ev.Env)
205 if err != nil || !ok {
206 return ok, err
207 }
208 }
209 if len(r.Changes) > 0 && ev.Changed != nil {
210 if !anyGlob(r.Changes, ev.Changed) {
211 return false, nil
212 }
213 }
214 if len(r.Exists) > 0 && ev.Files != nil {
215 if !allExist(r.Exists, ev.Files) {
216 return false, nil
217 }
218 }
219 if r.If == "" && len(r.Changes) == 0 && len(r.Exists) == 0 && r.When == "" {
220 return false, nil
221 }
222 return true, nil
223}
224
225func onlyExcept(j Job, ev Eval) (bool, error) {
226 mr := ev.Env["CI_PIPELINE_SOURCE"] == "merge_request_event"
227 if len(j.Only) == 0 && len(j.Except) == 0 {
228 // Zonder rules of only draait een job op een branch of tag,
229 // niet in een merge-request-pipeline.
230 if mr {
231 return false, nil
232 }
233 return true, nil
234 }
235 name := ev.Env["CI_COMMIT_REF_NAME"]
236 if name == "" {
237 name = ev.Env["CI_COMMIT_BRANCH"]
238 }
239 tag := ev.Env["CI_COMMIT_TAG"] != ""
240 if len(j.Only) > 0 {
241 ok, err := refListed(j.Only, name, tag, mr)
242 if err != nil || !ok {
243 return false, err
244 }
245 }
246 if len(j.Except) > 0 {
247 ok, err := refListed(j.Except, name, tag, mr)
248 if err != nil {
249 return false, err
250 }
251 if ok {
252 return false, nil
253 }
254 }
255 return true, nil
256}
257
258func refListed(list []string, name string, tag, mr bool) (bool, error) {
259 for _, item := range list {
260 switch item {
261 case "merge_requests":
262 if mr {
263 return true, nil
264 }
265 case "branches":
266 if !tag && !mr {
267 return true, nil
268 }
269 case "tags":
270 if tag && !mr {
271 return true, nil
272 }
273 default:
274 if mr {
275 continue
276 }
277 if strings.HasPrefix(item, "/") && strings.HasSuffix(item, "/") && len(item) > 2 {
278 re, err := regexp.Compile(item[1 : len(item)-1])
279 if err != nil {
280 return false, i18n.E("ci.err.regex", item, err.Error())
281 }
282 if re.MatchString(name) {
283 return true, nil
284 }
285 continue
286 }
287 if item == name {
288 return true, nil
289 }
290 }
291 }
292 return false, nil
293}
294
295func anyGlob(patterns, names []string) bool {
296 for _, name := range names {
297 for _, pattern := range patterns {
298 if Glob(pattern, name) {
299 return true
300 }
301 }
302 }
303 return false
304}
305
306func allExist(patterns, files []string) bool {
307 for _, pattern := range patterns {
308 found := false
309 for _, file := range files {
310 if file == pattern || Glob(pattern, file) {
311 found = true
312 break
313 }
314 }
315 if !found {
316 return false
317 }
318 }
319 return true
320}
321
322// Glob volgt GitLab-padpatronen: * blijft in één segment, ** loopt door mappen.
323func Glob(pattern, name string) bool {
324 pattern = strings.TrimPrefix(pattern, "/")
325 name = strings.TrimPrefix(name, "/")
326 if pattern == name {
327 return true
328 }
329 re := globRegexp(pattern)
330 if re.MatchString(name) {
331 return true
332 }
333 if !strings.Contains(pattern, "/") && re.MatchString(path.Base(name)) {
334 return true
335 }
336 return false
337}
338
339func globRegexp(pattern string) *regexp.Regexp {
340 var b strings.Builder
341 b.WriteString("^")
342 for i := 0; i < len(pattern); i++ {
343 if pattern[i] == '*' && i+1 < len(pattern) && pattern[i+1] == '*' {
344 b.WriteString(".*")
345 i++
346 if i+1 < len(pattern) && pattern[i+1] == '/' {
347 i++
348 }
349 continue
350 }
351 switch pattern[i] {
352 case '*':
353 b.WriteString("[^/]*")
354 case '?':
355 b.WriteString("[^/]")
356 default:
357 b.WriteString(regexp.QuoteMeta(pattern[i : i+1]))
358 }
359 }
360 b.WriteString("$")
361 return regexp.MustCompile(b.String())
362}
363
364// Gate beslist of een job mag starten.
365// "ready" betekent dat alle voorgangers klaar zijn.
366// "wait" betekent dat er nog een voorganger loopt of in de wachtrij staat.
367// "skip" betekent dat een voorganger mislukte zonder allow_failure,
368// of dat when: on_failure geen mislukking zag.
369// "fail" betekent dat een verplichte need niet in deze pipeline zit.
370//
371// NeedsSet volgt de genoemde jobs, ook als de lijst leeg is.
372// Zonder needs wacht de job op alle jobs in eerdere stages.
373func Gate(jobs []Job, status map[string]string, job Job) string {
374 byName := map[string]Job{}
375 for _, j := range jobs {
376 byName[j.Name] = j
377 }
378 optional := map[string]bool{}
379 for _, n := range job.Optional {
380 optional[n] = true
381 }
382 var deps []Job
383 if job.NeedsSet {
384 for _, n := range job.Needs {
385 d, ok := byName[n]
386 if !ok {
387 if optional[n] {
388 continue
389 }
390 return "fail"
391 }
392 deps = append(deps, d)
393 }
394 } else {
395 stagePos := map[string]int{}
396 order := 0
397 for _, j := range jobs {
398 if _, ok := stagePos[j.Stage]; !ok {
399 stagePos[j.Stage] = order
400 order++
401 }
402 }
403 my := stagePos[job.Stage]
404 for _, j := range jobs {
405 if stagePos[j.Stage] < my {
406 deps = append(deps, j)
407 }
408 }
409 }
410 failedAny := false
411 failedHard := false
412 for _, d := range deps {
413 switch status[d.Name] {
414 case "success", "skipped":
415 continue
416 case "failed", "canceled":
417 failedAny = true
418 if !d.AllowFailure {
419 failedHard = true
420 }
421 case "manual":
422 if d.AllowFailure {
423 continue
424 }
425 return "wait"
426 default:
427 return "wait"
428 }
429 }
430 when := job.When
431 if when == "" {
432 when = "on_success"
433 }
434 switch when {
435 case "always":
436 return "ready"
437 case "on_failure":
438 if failedAny {
439 return "ready"
440 }
441 return "skip"
442 default:
443 if failedHard {
444 return "skip"
445 }
446 return "ready"
447 }
448}
449
450func evalRule(expr string, env map[string]string) (bool, error) {
451 p := &ruleParser{s: expr, env: env}
452 ok, err := p.parseOr()
453 if err != nil {
454 return false, err
455 }
456 p.skip()
457 if p.i < len(p.s) {
458 return false, i18n.E("ci.err.ruleExpr", strings.TrimSpace(expr))
459 }
460 return ok, nil
461}
462
463const maxParallel = 50
464
465func expandJobs(jobs []Job) ([]Job, error) {
466 expanded := map[string][]string{}
467 var out []Job
468 seen := map[string]bool{}
469 for _, j := range jobs {
470 instances, err := expandOne(j)
471 if err != nil {
472 return nil, fmt.Errorf("job %s: %w", j.Name, err)
473 }
474 names := make([]string, len(instances))
475 for i, inst := range instances {
476 if seen[inst.Name] {
477 return nil, i18n.E("ci.err.jobDup", inst.Name)
478 }
479 seen[inst.Name] = true
480 names[i] = inst.Name
481 out = append(out, inst)
482 }
483 expanded[j.Name] = names
484 }
485 for i := range out {
486 out[i].Needs = rewriteNeeds(out[i].Needs, expanded)
487 out[i].Optional = rewriteNeeds(out[i].Optional, expanded)
488 }
489 return out, nil
490}
491
492func expandOne(j Job) ([]Job, error) {
493 if len(j.Matrix) > 0 {
494 var combos []matrixCombo
495 for _, spec := range j.Matrix {
496 combos = append(combos, cartesian(spec)...)
497 }
498 if len(combos) > maxParallel {
499 return nil, i18n.E("ci.err.parallel", len(combos), maxParallel)
500 }
501 if len(combos) == 0 {
502 return nil, i18n.E("ci.err.parallel", 0, maxParallel)
503 }
504 var out []Job
505 for i, combo := range combos {
506 cp := copyJob(j)
507 cp.Name = j.Name + ": [" + strings.Join(combo.values, ", ") + "]"
508 cp.Parallel = 0
509 cp.Matrix = nil
510 for k, v := range combo.vars {
511 cp.Variables[k] = v
512 }
513 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i + 1)
514 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(len(combos))
515 out = append(out, cp)
516 }
517 return out, nil
518 }
519 if j.Parallel <= 1 {
520 return []Job{j}, nil
521 }
522 if j.Parallel > maxParallel {
523 return nil, i18n.E("ci.err.parallel", j.Parallel, maxParallel)
524 }
525 var out []Job
526 for i := 1; i <= j.Parallel; i++ {
527 cp := copyJob(j)
528 cp.Name = fmt.Sprintf("%s %d/%d", j.Name, i, j.Parallel)
529 cp.Parallel = 0
530 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i)
531 cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(j.Parallel)
532 out = append(out, cp)
533 }
534 return out, nil
535}
536
537type matrixCombo struct {
538 values []string
539 vars map[string]string
540}
541
542func cartesian(spec MatrixSpec) []matrixCombo {
543 if len(spec.Keys) == 0 {
544 return nil
545 }
546 var out []matrixCombo
547 var walk func(i int, vals []string, vars map[string]string)
548 walk = func(i int, vals []string, vars map[string]string) {
549 if i == len(spec.Keys) {
550 cp := map[string]string{}
551 for k, v := range vars {
552 cp[k] = v
553 }
554 out = append(out, matrixCombo{values: append([]string{}, vals...), vars: cp})
555 return
556 }
557 for _, v := range spec.Vals[i] {
558 vars[spec.Keys[i]] = v
559 walk(i+1, append(vals, v), vars)
560 }
561 }
562 walk(0, nil, map[string]string{})
563 return out
564}
565
566func copyJob(j Job) Job {
567 cp := j
568 cp.Variables = map[string]string{}
569 for k, v := range j.Variables {
570 cp.Variables[k] = v
571 }
572 cp.Needs = append([]string{}, j.Needs...)
573 cp.Optional = append([]string{}, j.Optional...)
574 cp.Services = append([]Service{}, j.Services...)
575 return cp
576}
577
578func rewriteNeeds(list []string, expanded map[string][]string) []string {
579 if list == nil {
580 return nil
581 }
582 var out []string
583 for _, n := range list {
584 names := expanded[n]
585 if len(names) == 0 || (len(names) == 1 && names[0] == n) {
586 out = append(out, n)
587 continue
588 }
589 out = append(out, names...)
590 }
591 if out == nil {
592 return []string{}
593 }
594 return out
595}
596
597// Expand vervangt $NAAM en ${NAAM} in een tekst, bijvoorbeeld environment:url.
598func Expand(s string, env map[string]string) string {
599 var b strings.Builder
600 for i := 0; i < len(s); i++ {
601 if s[i] != '$' || i+1 >= len(s) {
602 b.WriteByte(s[i])
603 continue
604 }
605 if s[i+1] == '{' {
606 end := strings.IndexByte(s[i+2:], '}')
607 if end < 0 {
608 b.WriteByte('$')
609 continue
610 }
611 name := s[i+2 : i+2+end]
612 b.WriteString(env[name])
613 i += 2 + end
614 continue
615 }
616 j := i + 1
617 for j < len(s) && isIdent(s[j]) {
618 j++
619 }
620 if j == i+1 {
621 b.WriteByte('$')
622 continue
623 }
624 b.WriteString(env[s[i+1:j]])
625 i = j - 1
626 }
627 return b.String()
628}
629
630func contains(list []string, s string) bool {
631 for _, item := range list {
632 if item == s {
633 return true
634 }
635 }
636 return false
637}