package ci import ( "fmt" "path" "regexp" "sort" "strconv" "strings" "gitlab.com/Timfa/bluegit/internal/i18n" ) // Variables zet de voorgedefinieerde CI_-variabelen. Job-specifieke namen // vult de runner zelf aan. func Variables(r Ref) map[string]string { name := r.Name kind := r.Kind if kind == "" { name, kind = SplitRef(r.FullRef) } short := r.SHA if len(short) > 8 { short = short[:8] } project := r.ProjectPath ns, base := splitProject(project) env := map[string]string{ "CI": "true", "CI_CONFIG_PATH": "blue-ci-cd.yml", "CI_SERVER_NAME": "BlueGit", "CI_COMMIT_SHA": r.SHA, "CI_COMMIT_SHORT_SHA": short, "CI_COMMIT_REF": r.FullRef, "CI_COMMIT_REF_NAME": name, "CI_COMMIT_REF_SLUG": slug(name), "CI_COMMIT_TITLE": r.Title, "CI_COMMIT_BRANCH": "", "CI_COMMIT_TAG": "", "CI_DEFAULT_BRANCH": r.DefaultBranch, "CI_PIPELINE_ID": fmt.Sprint(r.PipelineID), "CI_PIPELINE_SOURCE": r.Source, "CI_PIPELINE_URL": strings.TrimRight(r.ProjectURL, "/") + "/-/pipelines/" + fmt.Sprint(r.PipelineID), "CI_PROJECT_PATH": project, "CI_PROJECT_NAME": base, "CI_PROJECT_NAMESPACE": ns, "CI_PROJECT_URL": r.ProjectURL, "CI_API_URL": r.APIURL, } if r.Timestamp != "" { env["CI_COMMIT_TIMESTAMP"] = r.Timestamp } if kind == "tag" { env["CI_COMMIT_TAG"] = name } else { env["CI_COMMIT_BRANCH"] = name } if env["CI_PIPELINE_SOURCE"] == "" { env["CI_PIPELINE_SOURCE"] = "push" } if r.MR != nil { m := r.MR env["CI_MERGE_REQUEST_ID"] = fmt.Sprint(m.ID) env["CI_MERGE_REQUEST_IID"] = fmt.Sprint(m.IID) env["CI_MERGE_REQUEST_TITLE"] = m.Title env["CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"] = m.SourceBranch env["CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"] = r.SHA env["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"] = m.TargetBranch env["CI_MERGE_REQUEST_TARGET_BRANCH_SHA"] = m.TargetSHA env["CI_MERGE_REQUEST_PROJECT_PATH"] = m.ProjectPath env["CI_MERGE_REQUEST_PROJECT_URL"] = m.ProjectURL } return env } // SplitRef haalt de korte naam en het soort uit refs/heads of refs/tags. func SplitRef(ref string) (name, kind string) { switch { case strings.HasPrefix(ref, "refs/tags/"): return strings.TrimPrefix(ref, "refs/tags/"), "tag" case strings.HasPrefix(ref, "refs/heads/"): return strings.TrimPrefix(ref, "refs/heads/"), "branch" default: return ref, "branch" } } func splitProject(path string) (ns, name string) { i := strings.LastIndex(path, "/") if i < 0 { return "", path } return path[:i], path[i+1:] } func slug(s string) string { var b strings.Builder prev := false for _, r := range strings.ToLower(s) { if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { b.WriteRune(r) prev = false continue } if !prev && b.Len() > 0 { b.WriteByte('-') prev = true } } return strings.Trim(b.String(), "-") } // Select houdt de jobs waarvan de rules, only en except kloppen. // De When van de job wordt de when van de winnende rule. func (p Pipeline) Select(ev Eval) ([]Job, error) { if ev.Env == nil { ev.Env = map[string]string{} } var out []Job active := map[string]bool{} var pending []Job for _, j := range p.Jobs { ok, when, err := j.matches(ev) if err != nil { return nil, fmt.Errorf("job %s: %w", j.Name, err) } if !ok { continue } if when == "" { when = "on_success" } j.When = when pending = append(pending, j) active[j.Name] = true } for i := range pending { var needs []string for _, n := range pending[i].Needs { if active[n] { needs = append(needs, n) continue } if contains(pending[i].Optional, n) { continue } needs = append(needs, n) } if needs == nil { needs = []string{} } pending[i].Needs = needs out = append(out, pending[i]) } expanded, err := expandJobs(out) if err != nil { return nil, err } return p.byStage(expanded), nil } // byStage zet jobs in de volgorde van stages:, stabiel binnen een stage. // Gate en de runner leiden de stagevolgorde af uit de volgorde van de // slice. Een job die in het bestand vóór een eerdere stage staat, start // daardoor niet eerder. func (p Pipeline) byStage(jobs []Job) []Job { if len(p.Stages) == 0 || len(jobs) < 2 { return jobs } rank := map[string]int{} for i, stage := range p.Stages { rank[stage] = i } sort.SliceStable(jobs, func(i, j int) bool { ri, okI := rank[jobs[i].Stage] rj, okJ := rank[jobs[j].Stage] if !okI { ri = len(p.Stages) } if !okJ { rj = len(p.Stages) } return ri < rj }) return jobs } // Allow zegt of workflow:rules deze pipeline toelaat. // Zonder workflow wordt de pipeline gemaakt. Een passende rule met when:never // of geen enkele passende rule slaat de pipeline over. func (p Pipeline) Allow(ev Eval) (bool, error) { if len(p.Workflow) == 0 { return true, nil } if ev.Env == nil { ev.Env = map[string]string{} } ok, _, err := (Job{Rules: p.Workflow, When: "always"}).matches(ev) return ok, err } // ActiveJobs is Select met alleen variabelen. changes en exists gelden dan als waar. func (p Pipeline) ActiveJobs(env map[string]string) ([]Job, error) { return p.Select(Eval{Env: env}) } func (j Job) matches(ev Eval) (bool, string, error) { when := j.When if when == "" { when = "on_success" } if len(j.Rules) == 0 { ok, err := onlyExcept(j, ev) return ok, when, err } for _, r := range j.Rules { ok, err := ruleMatch(r, ev) if err != nil { return false, "", err } if !ok { continue } w := r.When if w == "" { w = when } if w == "never" { return false, w, nil } return true, w, nil } return false, "", nil } func ruleMatch(r Rule, ev Eval) (bool, error) { if r.If != "" { ok, err := evalRule(r.If, ev.Env) if err != nil || !ok { return ok, err } } if len(r.Changes) > 0 && ev.Changed != nil { if !anyGlob(r.Changes, ev.Changed) { return false, nil } } if len(r.Exists) > 0 && ev.Files != nil { if !allExist(r.Exists, ev.Files) { return false, nil } } if r.If == "" && len(r.Changes) == 0 && len(r.Exists) == 0 && r.When == "" { return false, nil } return true, nil } func onlyExcept(j Job, ev Eval) (bool, error) { mr := ev.Env["CI_PIPELINE_SOURCE"] == "merge_request_event" if len(j.Only) == 0 && len(j.Except) == 0 { // Zonder rules of only draait een job op een branch of tag, // niet in een merge-request-pipeline. if mr { return false, nil } return true, nil } name := ev.Env["CI_COMMIT_REF_NAME"] if name == "" { name = ev.Env["CI_COMMIT_BRANCH"] } tag := ev.Env["CI_COMMIT_TAG"] != "" if len(j.Only) > 0 { ok, err := refListed(j.Only, name, tag, mr) if err != nil || !ok { return false, err } } if len(j.Except) > 0 { ok, err := refListed(j.Except, name, tag, mr) if err != nil { return false, err } if ok { return false, nil } } return true, nil } func refListed(list []string, name string, tag, mr bool) (bool, error) { for _, item := range list { switch item { case "merge_requests": if mr { return true, nil } case "branches": if !tag && !mr { return true, nil } case "tags": if tag && !mr { return true, nil } default: if mr { continue } if strings.HasPrefix(item, "/") && strings.HasSuffix(item, "/") && len(item) > 2 { re, err := regexp.Compile(item[1 : len(item)-1]) if err != nil { return false, i18n.E("ci.err.regex", item, err.Error()) } if re.MatchString(name) { return true, nil } continue } if item == name { return true, nil } } } return false, nil } func anyGlob(patterns, names []string) bool { for _, name := range names { for _, pattern := range patterns { if Glob(pattern, name) { return true } } } return false } func allExist(patterns, files []string) bool { for _, pattern := range patterns { found := false for _, file := range files { if file == pattern || Glob(pattern, file) { found = true break } } if !found { return false } } return true } // Glob volgt GitLab-padpatronen: * blijft in één segment, ** loopt door mappen. func Glob(pattern, name string) bool { pattern = strings.TrimPrefix(pattern, "/") name = strings.TrimPrefix(name, "/") if pattern == name { return true } re := globRegexp(pattern) if re.MatchString(name) { return true } if !strings.Contains(pattern, "/") && re.MatchString(path.Base(name)) { return true } return false } func globRegexp(pattern string) *regexp.Regexp { var b strings.Builder b.WriteString("^") for i := 0; i < len(pattern); i++ { if pattern[i] == '*' && i+1 < len(pattern) && pattern[i+1] == '*' { b.WriteString(".*") i++ if i+1 < len(pattern) && pattern[i+1] == '/' { i++ } continue } switch pattern[i] { case '*': b.WriteString("[^/]*") case '?': b.WriteString("[^/]") default: b.WriteString(regexp.QuoteMeta(pattern[i : i+1])) } } b.WriteString("$") return regexp.MustCompile(b.String()) } // Gate beslist of een job mag starten. // "ready" betekent dat alle voorgangers klaar zijn. // "wait" betekent dat er nog een voorganger loopt of in de wachtrij staat. // "skip" betekent dat een voorganger mislukte zonder allow_failure, // dat een job in needs is overgeslagen, of dat when: on_failure geen // mislukking zag. // "fail" betekent dat een verplichte need niet in deze pipeline zit. // // NeedsSet volgt de genoemde jobs, ook als de lijst leeg is. Een // gefaalde of overgeslagen need slaat deze job over, behalve bij // allow_failure, when: on_failure en when: always. needs: [] heeft // geen voorgangers en start meteen. // Zonder needs wacht de job op alle jobs in eerdere stages. Een // overgeslagen eerdere stage telt dan als klaar; een harde mislukking // houdt latere stages tegen. De stagevolgorde is de volgorde waarin // de stages in de slice voor het eerst voorkomen. Select sorteert op // stages:, zodat een latere stage die bovenaan het bestand staat niet // eerder ready wordt. func Gate(jobs []Job, status map[string]string, job Job) string { byName := map[string]Job{} for _, j := range jobs { byName[j.Name] = j } optional := map[string]bool{} for _, n := range job.Optional { optional[n] = true } var deps []Job if job.NeedsSet { for _, n := range job.Needs { d, ok := byName[n] if !ok { if optional[n] { continue } return "fail" } deps = append(deps, d) } } else { stagePos := map[string]int{} order := 0 for _, j := range jobs { if _, ok := stagePos[j.Stage]; !ok { stagePos[j.Stage] = order order++ } } my := stagePos[job.Stage] for _, j := range jobs { if stagePos[j.Stage] < my { deps = append(deps, j) } } } failedAny := false failedHard := false skippedNeed := false for _, d := range deps { switch status[d.Name] { case "success": continue case "skipped": // Alleen een genoemde need. Zonder needs telt een // overgeslagen eerdere stage als afgerond. if job.NeedsSet { skippedNeed = true } case "failed", "canceled": failedAny = true if !d.AllowFailure { failedHard = true } case "manual": if d.AllowFailure { continue } return "wait" default: return "wait" } } when := job.When if when == "" { when = "on_success" } switch when { case "always": return "ready" case "on_failure": if failedAny { return "ready" } return "skip" default: if failedHard || skippedNeed { return "skip" } return "ready" } } func evalRule(expr string, env map[string]string) (bool, error) { p := &ruleParser{s: expr, env: env} ok, err := p.parseOr() if err != nil { return false, err } p.skip() if p.i < len(p.s) { return false, i18n.E("ci.err.ruleExpr", strings.TrimSpace(expr)) } return ok, nil } const maxParallel = 50 func expandJobs(jobs []Job) ([]Job, error) { expanded := map[string][]string{} var out []Job seen := map[string]bool{} for _, j := range jobs { instances, err := expandOne(j) if err != nil { return nil, fmt.Errorf("job %s: %w", j.Name, err) } names := make([]string, len(instances)) for i, inst := range instances { if seen[inst.Name] { return nil, i18n.E("ci.err.jobDup", inst.Name) } seen[inst.Name] = true names[i] = inst.Name out = append(out, inst) } expanded[j.Name] = names } for i := range out { out[i].Needs = rewriteNeeds(out[i].Needs, expanded) out[i].Optional = rewriteNeeds(out[i].Optional, expanded) } return out, nil } func expandOne(j Job) ([]Job, error) { if len(j.Matrix) > 0 { var combos []matrixCombo for _, spec := range j.Matrix { combos = append(combos, cartesian(spec)...) } if len(combos) > maxParallel { return nil, i18n.E("ci.err.parallel", len(combos), maxParallel) } if len(combos) == 0 { return nil, i18n.E("ci.err.parallel", 0, maxParallel) } var out []Job for i, combo := range combos { cp := copyJob(j) cp.Name = j.Name + ": [" + strings.Join(combo.values, ", ") + "]" cp.Parallel = 0 cp.Matrix = nil for k, v := range combo.vars { cp.Variables[k] = v } cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i + 1) cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(len(combos)) out = append(out, cp) } return out, nil } if j.Parallel <= 1 { return []Job{j}, nil } if j.Parallel > maxParallel { return nil, i18n.E("ci.err.parallel", j.Parallel, maxParallel) } var out []Job for i := 1; i <= j.Parallel; i++ { cp := copyJob(j) cp.Name = fmt.Sprintf("%s %d/%d", j.Name, i, j.Parallel) cp.Parallel = 0 cp.Variables["CI_NODE_INDEX"] = strconv.Itoa(i) cp.Variables["CI_NODE_TOTAL"] = strconv.Itoa(j.Parallel) out = append(out, cp) } return out, nil } type matrixCombo struct { values []string vars map[string]string } func cartesian(spec MatrixSpec) []matrixCombo { if len(spec.Keys) == 0 { return nil } var out []matrixCombo var walk func(i int, vals []string, vars map[string]string) walk = func(i int, vals []string, vars map[string]string) { if i == len(spec.Keys) { cp := map[string]string{} for k, v := range vars { cp[k] = v } out = append(out, matrixCombo{values: append([]string{}, vals...), vars: cp}) return } for _, v := range spec.Vals[i] { vars[spec.Keys[i]] = v walk(i+1, append(vals, v), vars) } } walk(0, nil, map[string]string{}) return out } func copyJob(j Job) Job { cp := j cp.Variables = map[string]string{} for k, v := range j.Variables { cp.Variables[k] = v } cp.Needs = append([]string{}, j.Needs...) cp.Optional = append([]string{}, j.Optional...) cp.Services = append([]Service{}, j.Services...) return cp } func rewriteNeeds(list []string, expanded map[string][]string) []string { if list == nil { return nil } var out []string for _, n := range list { names := expanded[n] if len(names) == 0 || (len(names) == 1 && names[0] == n) { out = append(out, n) continue } out = append(out, names...) } if out == nil { return []string{} } return out } // Expand vervangt $NAAM en ${NAAM} in een tekst, bijvoorbeeld environment:url. func Expand(s string, env map[string]string) string { var b strings.Builder for i := 0; i < len(s); i++ { if s[i] != '$' || i+1 >= len(s) { b.WriteByte(s[i]) continue } if s[i+1] == '{' { end := strings.IndexByte(s[i+2:], '}') if end < 0 { b.WriteByte('$') continue } name := s[i+2 : i+2+end] b.WriteString(env[name]) i += 2 + end continue } j := i + 1 for j < len(s) && isIdent(s[j]) { j++ } if j == i+1 { b.WriteByte('$') continue } b.WriteString(env[s[i+1:j]]) i = j - 1 } return b.String() } func contains(list []string, s string) bool { for _, item := range list { if item == s { return true } } return false }