// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
package runner

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"gitlab.com/Timfa/bluegit/internal/access"
	"gitlab.com/Timfa/bluegit/internal/ci"
	"gitlab.com/Timfa/bluegit/internal/gitrepo"
	"gitlab.com/Timfa/bluegit/internal/images"
	"gitlab.com/Timfa/bluegit/internal/runnercfg"
	"gitlab.com/Timfa/bluegit/internal/store"
	"gitlab.com/Timfa/bluegit/internal/webhook"
)

// Runner plant jobs en stuurt het resultaat door naar webhooks.
type Runner struct {
	// Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
	Hold    func(store.Pipeline) bool
	Store   *store.Store
	GitBin  string
	Docker  string
	Storage string
	BaseURL string
	Network string
	Name    string
	Timeout time.Duration
	Log     *slog.Logger
	OnEvent func(repoID int64, ev webhook.Event)
	// Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
	Presets map[string]images.Override
	// Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
	Jobs     *runnercfg.Tracker
	ctl      control
	slots    chan struct{}
	mu       sync.RWMutex
	cancelMu sync.Mutex
	// protected telt lopende doNotInterrupt-jobs van de ingebouwde runner.
	protected atomic.Int64
	cancel    map[int64]context.CancelFunc
}

// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
func (r *Runner) Interrupt(ids []int64) {
	r.cancelMu.Lock()
	defer r.cancelMu.Unlock()
	for _, id := range ids {
		if c := r.cancel[id]; c != nil {
			c()
		}
	}
}

// cancelReason leest waarom een lopende job gestopt werd. CancelJob (door
// een gebruiker) zet canceled_by_user al in de database; anders was het
// een nieuwere pipeline op dezelfde ref.
func (r *Runner) cancelReason(repoID, id int64) string {
	if j, err := r.Store.JobByID(context.Background(), repoID, id); err == nil && j.Reason == "canceled_by_user" {
		return j.Reason
	}
	return "canceled_by_newer_pipeline"
}

func cancelText(reason string) string {
	if reason == "canceled_by_user" {
		return "canceled by a user"
	}
	return "canceled because a newer pipeline started for the same ref"
}

func (r *Runner) track(id int64, cancel context.CancelFunc) {
	r.cancelMu.Lock()
	defer r.cancelMu.Unlock()
	if r.cancel == nil {
		r.cancel = map[int64]context.CancelFunc{}
	}
	r.cancel[id] = cancel
}

func (r *Runner) untrack(id int64) {
	r.cancelMu.Lock()
	defer r.cancelMu.Unlock()
	delete(r.cancel, id)
}

// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
func (r *Runner) SetStorage(path string) {
	r.mu.Lock()
	r.Storage = path
	r.mu.Unlock()
}

func (r *Runner) storage() string {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.Storage
}

// Loop blijft jobs oppakken tot ctx afloopt.
func (r *Runner) Loop(ctx context.Context, concurrency int) {
	if concurrency < 1 {
		concurrency = 1
	}
	r.slots = make(chan struct{}, maxSlots)
	r.mu.Lock()
	if !r.ctl.set {
		r.ctl.limit = concurrency
	}
	r.mu.Unlock()
	tick := time.NewTicker(2 * time.Second)
	defer tick.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
			r.Tick(ctx)
		}
	}
}

// Tick bekijkt alle open pipelines één keer.
// Zonder opslagpad gebeurt er niets.
func (r *Runner) Tick(ctx context.Context) {
	if r.storage() == "" {
		return
	}
	pipes, err := r.Store.ActivePipelines(ctx)
	if err != nil {
		r.Log.Error("pipelines", "err", err)
		return
	}
	for _, p := range pipes {
		r.schedule(ctx, p)
	}
	r.reapRemote(ctx)
}

func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
	r.settle(ctx, p.ID)
	if p.Status == "running" {
		// Een pipeline die alleen nog op handmatige jobs wacht, is niet
		// meer bezig (ook oude pipelines van voor deze regel).
		if st, err := r.Store.RefreshPipelineStatus(ctx, p.ID); err == nil {
			p.Status = st
		}
	}
	if r.Hold != nil && r.Hold(p) {
		return // wacht op de self-update
	}
	jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
	if err != nil {
		r.Log.Error("jobs", "err", err)
		return
	}
	specs := make([]ci.Job, len(jobs))
	status := map[string]string{}
	for i, j := range jobs {
		specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
		status[j.Name] = j.Status
	}
	tierCache := map[int64][]Tier{}
	for _, j := range jobs {
		if j.Status != "queued" && j.Status != "manual" {
			continue
		}
		switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
		case "skip":
			_ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
			status[j.Name] = "skipped"
		case "fail":
			_ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
			status[j.Name] = "failed"
		case "ready":
			if j.Status != "queued" {
				continue
			}
			if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
				continue
			}
			// Meer jobs dan ci.concurrency blijven queued, met een plek
			// in de wachtrij. Het slot gaat pas vrij als execute klaar is.
			free := r.acquireSlot()
			prot := r.refProtected(ctx, j.RepoID, p.Ref)
			if prot {
				r.noteProtected(ctx, j.RepoID, j)
			}
			if tiers := r.tiersFor(ctx, tierCache, j.RepoID, prot); len(tiers) > 1 {
				jr := r.Store.JobRun(ctx, j.ID)
				step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
				if !step.Builtin {
					if free {
						r.releaseSlot()
					}
					if step.Moved {
						_ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
					}
					continue
				}
			}
			if !free {
				continue
			}
			ok, err := r.Store.MarkJobRunning(ctx, j.ID)
			if err != nil || !ok {
				r.releaseSlot()
				continue
			}
			_ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
			_ = r.Store.ClearJobOffer(ctx, j.ID)
			status[j.Name] = "running"
			job := j
			pipe := p
			go r.execute(ctx, pipe, job)
		}
	}
}

// settle slaat jobs over die door een harde mislukking of een
// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
// pipelinestatus al op failed was gezet.
func (r *Runner) settle(ctx context.Context, pipelineID int64) {
	for {
		jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
		if err != nil {
			if r.Log != nil {
				r.Log.Error("jobs", "err", err)
			}
			return
		}
		specs := make([]ci.Job, len(jobs))
		status := map[string]string{}
		for i, j := range jobs {
			specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
			status[j.Name] = j.Status
		}
		moved := false
		for _, j := range jobs {
			if j.Status != "queued" && j.Status != "manual" {
				continue
			}
			switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
			case "skip":
				if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
					r.Log.Error("skip", "err", err)
					continue
				}
				status[j.Name] = "skipped"
				moved = true
			case "fail":
				if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
					r.Log.Error("fail", "err", err)
					continue
				}
				status[j.Name] = "failed"
				moved = true
			}
		}
		if !moved {
			return
		}
	}
}

// BuiltinName is de naam van de ingebouwde runner in de job-UI.
func (r *Runner) BuiltinName() string {
	if r.Name != "" {
		return r.Name
	}
	return "BlueGit runner"
}

func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64, protected bool) []Tier {
	key := repoID * 2
	if protected {
		key++
	}
	if t, ok := cache[key]; ok {
		return t
	}
	var tiers []Tier
	if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
		tiers, _ = Tiers(ctx, r.Store, repo, protected)
	}
	cache[key] = tiers
	return tiers
}

// refProtected: draait een pipeline op een beschermde ref (en krijgt
// hij dus beschermde geheimen)?
func (r *Runner) refProtected(ctx context.Context, repoID int64, ref string) bool {
	prot, err := r.refIsProtected(ctx, repoID, ref)
	if err != nil {
		return true // bij twijfel streng
	}
	return prot
}

// refIsProtected: een branch is beschermd als hij op een beschermd
// branchpatroon past, een tag als hij op een beschermd tagpatroon past
// (alleen maintainers mogen zo'n tag pushen). Een tag telt nooit via een
// branchpatroon, een branch nooit via een tagpatroon.
func (r *Runner) refIsProtected(ctx context.Context, repoID int64, ref string) (bool, error) {
	name, kind := ci.SplitRef(ref)
	if kind == "tag" {
		patterns, err := r.Store.ProtectedTagPatterns(ctx, repoID)
		if err != nil {
			return false, err
		}
		return access.BranchProtected(patterns, name), nil
	}
	patterns, err := r.Store.ProtectedPatterns(ctx, repoID)
	if err != nil {
		return false, err
	}
	return access.BranchProtected(patterns, name), nil
}

var protectedNoted sync.Map // job id → melding al in het log

// noteProtected zet één keer een melding in het joblog als er opt-in
// runners zijn die deze job niet mogen draaien.
func (r *Runner) noteProtected(ctx context.Context, repoID int64, job store.Job) {
	if _, done := protectedNoted.LoadOrStore(job.ID, true); done {
		return
	}
	repo, err := r.Store.RepoByID(ctx, repoID)
	if err != nil {
		return
	}
	if names := ExcludedForProtected(ctx, r.Store, repo); len(names) > 0 {
		_ = r.Store.AppendJobLog(ctx, job.ID, "Protected branch: this job only runs on the built-in runner, global runners, group runners and account runners of maintainers. Skipped: "+strings.Join(names, ", ")+"\n")
	}
}

func indexOf(jobs []store.Job, name string) int {
	for i, j := range jobs {
		if j.Name == name {
			return i
		}
	}
	return 0
}

func (r *Runner) acquireSlot() bool {
	if r.Paused() {
		return false
	}
	if r.slots == nil {
		return true
	}
	if l := r.controls().limit; l > 0 && len(r.slots) >= l {
		return false
	}
	select {
	case r.slots <- struct{}{}:
		return true
	default:
		return false
	}
}

func (r *Runner) releaseSlot() {
	if r.slots == nil {
		return
	}
	select {
	case <-r.slots:
	default:
	}
}

func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
	defer r.releaseSlot()
	before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
	parent := ctx
	if job.DoNotInterrupt {
		// Een herstart of stop van BlueGit breekt deze job niet af; main
		// wacht met afsluiten tot hij klaar is (zie WaitProtected).
		parent = context.WithoutCancel(ctx)
		r.protected.Add(1)
		defer r.protected.Add(-1)
	}
	base, stop := context.WithCancel(parent)
	r.track(job.ID, stop)
	defer func() {
		r.untrack(job.ID)
		stop()
	}()
	runCtx := base
	limit, source := EffectiveTimeout(job.Timeout, r.Timeout, r.controls().timeout)
	r.trackStart(job)
	if limit > 0 {
		var cancel context.CancelFunc
		runCtx, cancel = context.WithTimeout(base, limit)
		defer cancel()
	}
	mask, err := r.runOne(runCtx, p, job)
	status := "success"
	extra := ""
	kind := ""
	switch {
	case err == nil:
	case parent.Err() == nil && base.Err() != nil:
		status = "canceled"
		kind = r.cancelReason(job.RepoID, job.ID)
		extra = stamp(cancelText(kind))
	case errors.Is(runCtx.Err(), context.DeadlineExceeded):
		status = "failed"
		kind = "job_execution_timeout"
		extra = stamp(TimedOutLine(limit, source))
	default:
		status = "failed"
		kind = classifyFailure(err)
		extra = stamp(ci.Mask(err.Error(), mask))
	}
	r.trackDone(job.ID, status)
	r.complete(p, job, before, status, extra, kind)
}

// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
// en webhook. Gedeeld door de ingebouwde en de aparte runner.
func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
	if status == "failed" && shouldRetry(job, kind) {
		note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
		_ = r.Store.ClearJobOffer(context.Background(), job.ID)
		if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
			r.Log.Error("job opnieuw", "err", ferr)
		} else if ferr == nil {
			r.settle(context.Background(), p.ID)
			if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
				r.Log.Error("pipelinestatus", "err", rerr)
			}
			return
		}
	}
	if ferr := r.Store.FinishJobReason(context.Background(), job.ID, status, extra, kind); ferr != nil {
		r.Log.Error("job afronden", "err", ferr)
	}
	r.settle(context.Background(), p.ID)
	final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
	if rerr != nil {
		r.Log.Error("pipelinestatus", "err", rerr)
		return
	}
	if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
		kind := webhook.EventPipelineSuccess
		word := "geslaagd"
		if final == "failed" {
			kind = webhook.EventPipelineFailure
			word = "mislukt"
		}
		repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
		if err != nil {
			return
		}
		branch, _ := gitrepo.BranchOfRef(p.Ref)
		if branch == "" {
			branch = strings.TrimPrefix(p.Ref, "refs/heads/")
		}
		r.OnEvent(p.RepoID, webhook.Event{
			Type:       kind,
			Project:    repo.FullPath(),
			Branch:     branch,
			Commit:     p.CommitSHA,
			Summary:    fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
			URL:        strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
			PipelineID: fmt.Sprint(p.ID),
		})
	}
}

func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
	repo, err := r.Store.RepoByID(ctx, p.RepoID)
	if err != nil {
		return nil, err
	}
	sink := &logSink{store: r.Store, jobID: job.ID}
	sink.section("prepare", "Preparing the docker executor")
	work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
	if err := clearWorkDir(work); err != nil {
		sink.closeSection()
		return nil, fmt.Errorf("runner_system_failure: work dir: %w", err)
	}
	if err := os.MkdirAll(work, 0o755); err != nil {
		sink.closeSection()
		return nil, err
	}
	// Na afloop (ook bij falen of annuleren) gaat de werkmap weg. Artifacts
	// en cache staan dan al gekopieerd buiten de werkmap (collect, saveCache)
	// en de container is gestopt. Met cleanup_work uit blijft hij staan.
	defer r.finishWorkDir(work)
	bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
	if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
		sink.closeSection()
		return nil, fmt.Errorf("clone: %w", err)
	}
	// origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
	// op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
	if r.BaseURL != "" {
		_, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
	}
	if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
		sink.closeSection()
		return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
	}
	if err := r.restoreCache(repo.ID, job, work); err != nil {
		sink.closeSection()
		return nil, err
	}
	if err := r.materialize(ctx, p, job, work); err != nil {
		sink.closeSection()
		return nil, err
	}
	scriptPath := filepath.Join(work, ".bluegit-job.sh")
	host, _ := os.Hostname()
	script := shellScript(job, placeNote{
		Runner: r.Name, Host: host, WorkDir: "/workspace",
		PipelineID: p.ID, JobID: job.ID,
	})
	if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
		sink.closeSection()
		return nil, err
	}
	token, err := r.Store.CreateJobToken(ctx, job.ID)
	if err != nil {
		sink.closeSection()
		return nil, err
	}
	env, mask, err := r.environment(ctx, repo, p, job, token)
	if err != nil {
		sink.closeSection()
		return nil, err
	}
	sink.secrets = mask
	if err := r.execContainer(ctx, sink, job, work, env); err != nil {
		return mask, err
	}
	if err := r.collect(ctx, job, work); err != nil {
		return mask, err
	}
	if err := r.saveCache(repo.ID, job, work); err != nil {
		return mask, err
	}
	return mask, nil
}

// execContainer haalt het image op, start services en draait het
// jobscript in de container. De sectie prepare moet open staan.
func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
	ref, lines, rerr := r.resolveImage(ctx, job.Image)
	for _, line := range lines {
		sink.emit(line)
	}
	if rerr != nil {
		sink.closeSection()
		return rerr
	}
	job.Image = ref
	job.Services = append([]store.Service(nil), job.Services...)
	for i := range job.Services {
		sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
		for _, line := range slines {
			sink.emit(line)
		}
		if serr != nil {
			sink.closeSection()
			return serr
		}
		job.Services[i].Name = sref
	}
	network := r.Network
	if network == "" {
		network = "bridge"
	}
	if len(job.Services) > 0 {
		net, cleanup, serr := r.startServices(ctx, job)
		if serr != nil {
			sink.closeSection()
			return serr
		}
		defer cleanup()
		network = net
	}
	sink.closeSection()
	// -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
	// deze client overnemen. Zo blijft een meerregelige sleutel heel en
	// staat de waarde niet in de proceslijst.
	// Een vaste naam per job: een geannuleerde of afgebroken container
	// (de docker-client stopt dan, de container niet) wordt voor de start en
	// na afloop opgeruimd.
	cname := fmt.Sprintf("bluegit-job-%d", job.ID)
	removeContainer(r.Docker, cname)
	defer removeContainer(r.Docker, cname)
	args := []string{"run", "--rm", "--name", cname, "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
	ctl := r.controls()
	args = append(args, ctl.runArgs...)
	if ctl.pull == runnercfg.PullNever {
		args = append(args, "--pull", "never")
	}
	cmdEnv := append([]string{}, os.Environ()...)
	for k, v := range env {
		args = append(args, "-e", k)
		cmdEnv = append(cmdEnv, k+"="+v)
	}
	args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
	cmd := exec.CommandContext(ctx, r.Docker, args...)
	cmd.Env = cmdEnv
	cmd.Stdout = sink
	cmd.Stderr = sink
	err := cmd.Run()
	sink.flush()
	if err != nil {
		return fmt.Errorf("container ended with an error: %w", err)
	}
	return nil
}

func removeContainer(docker, name string) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	_, _ = execOut(ctx, docker, "rm", "-f", name)
}

// finishWorkDir ruimt de werkmap van een afgelopen job op, plus de lege
// pipelinemap erboven, tenzij de instelling cleanup_work uit staat.
func (r *Runner) finishWorkDir(work string) {
	if r.controls().keep {
		return
	}
	_ = clearWorkDir(work)
	_ = os.Remove(filepath.Dir(work)) // alleen als hij leeg is
}

// ClearWorkDir is clearWorkDir voor de aparte runner.
func ClearWorkDir(work string) error { return clearWorkDir(work) }

// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
// een lege map begint. Alleen als ook dat faalt, is het een fout.
func clearWorkDir(work string) error {
	if err := os.RemoveAll(work); err == nil {
		return nil
	}
	_ = filepath.WalkDir(work, func(p string, d fs.DirEntry, err error) error {
		if err == nil && d.IsDir() {
			_ = os.Chmod(p, 0o755)
		}
		return nil
	})
	if err := os.RemoveAll(work); err == nil {
		return nil
	}
	stale := fmt.Sprintf("%s.stale-%d", work, time.Now().UnixNano())
	if err := os.Rename(work, stale); err != nil {
		return err
	}
	go func() { _ = os.RemoveAll(stale) }()
	return nil
}

// placeNote zijn de vaste regels in de sectie Preparing environment.
type placeNote struct {
	Runner     string
	Host       string
	WorkDir    string
	PipelineID int64
	JobID      int64
}

func (n placeNote) lines() []string {
	runner := strings.TrimSpace(n.Runner)
	if runner == "" {
		runner = "Steve"
	}
	host := strings.TrimSpace(n.Host)
	if host == "" {
		host = "localhost"
	}
	dir := strings.TrimSpace(n.WorkDir)
	if dir == "" {
		dir = "/workspace"
	}
	return []string{
		fmt.Sprintf("Running on %s via %s...", runner, host),
		fmt.Sprintf("Working directory: %s", dir),
		fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
	}
}

func shSingle(s string) string {
	return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
}

func shellScript(job store.Job, note placeNote) string {
	var b strings.Builder
	b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
	fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
	for _, line := range note.lines() {
		fmt.Fprintf(&b, "echo %s\n", shSingle(line))
	}
	if strings.TrimSpace(job.Before) != "" {
		b.WriteString("(\nset -e\n")
		b.WriteString(job.Before)
		if !strings.HasSuffix(job.Before, "\n") {
			b.WriteByte('\n')
		}
		b.WriteString(")\nbrc=$?\n")
	} else {
		b.WriteString("brc=0\n")
	}
	b.WriteString("echo \"section_end:before:Preparing environment\"\n")
	b.WriteString("echo \"section_start:script:Running script\"\n")
	b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
	b.WriteString(job.Script)
	if !strings.HasSuffix(job.Script, "\n") {
		b.WriteByte('\n')
	}
	b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
	b.WriteString("echo \"section_end:script:Running script\"\n")
	b.WriteString("echo \"section_start:after:After script\"\n")
	if strings.TrimSpace(job.After) != "" {
		b.WriteString("(\nset -e\n")
		b.WriteString(job.After)
		if !strings.HasSuffix(job.After, "\n") {
			b.WriteByte('\n')
		}
		b.WriteString(")\n")
	}
	b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
	return b.String()
}

// resolveImage kiest het image waarmee de container start.
// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
	image = strings.TrimSpace(image)
	set, err := images.Effective(r.presetSet())
	if err != nil {
		return image, nil, fmt.Errorf("runner_system_failure: %w", err)
	}
	missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
	if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
		if p, ok := images.Find(set, image); ok {
			return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
		}
		return image, dockerImageLines(image, "", nil), missing
	}
	p, ok := images.Find(set, image)
	if !ok {
		if r.controls().pull == runnercfg.PullAlways {
			_, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
		}
		id, digests := r.inspectImage(ctx, image)
		return image, dockerImageLines(image, id, digests), nil
	}
	ref, built, err := p.Ensure(ctx, r.Docker)
	if err != nil {
		return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
	}
	var lines []string
	if built {
		lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
	}
	id, digests := r.inspectImage(ctx, ref)
	var baseDigests []string
	if ref != p.From {
		_, baseDigests = r.inspectImage(ctx, p.From)
	}
	lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
	return ref, lines, nil
}

func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
	var lines []string
	if preset != "" {
		lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
	}
	if ref == "" {
		return lines
	}
	lines = append(lines, dockerImageLines(ref, id, digests)...)
	if preset != "" && ref != base {
		digest := matchingDigest(base, baseDigests)
		if digest == "" && len(baseDigests) > 0 {
			digest = strings.TrimSpace(baseDigests[0])
		}
		if digest != "" {
			lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
		} else {
			lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
		}
	}
	return lines
}

func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
	if strings.TrimSpace(image) == "" || r.Docker == "" {
		return "", nil
	}
	out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
	if err != nil {
		return "", nil
	}
	return parseImageInspect(out)
}

func parseImageInspect(out string) (string, []string) {
	lines := strings.Split(out, "\n")
	id := ""
	if len(lines) > 0 {
		id = strings.TrimSpace(lines[0])
	}
	var digests []string
	for _, line := range lines[1:] {
		line = strings.TrimSpace(line)
		if line != "" {
			digests = append(digests, line)
		}
	}
	return id, digests
}

func dockerImageLines(image, id string, digests []string) []string {
	image = strings.TrimSpace(image)
	lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
	id = strings.TrimSpace(id)
	digest := matchingDigest(image, digests)
	switch {
	case id != "" && digest != "":
		lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
	case digest != "":
		lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
	case id != "":
		lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
	}
	return lines
}

func matchingDigest(image string, digests []string) string {
	name := imageName(image)
	fallback := ""
	for _, digest := range digests {
		digest = strings.TrimSpace(digest)
		at := strings.LastIndex(digest, "@")
		if digest == "" || at <= 0 {
			continue
		}
		if fallback == "" {
			fallback = digest
		}
		repo := digest[:at]
		if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
			return digest
		}
	}
	return fallback
}

func imageName(image string) string {
	image = strings.TrimSpace(image)
	if i := strings.LastIndex(image, "@"); i >= 0 {
		image = image[:i]
	}
	slash := strings.LastIndex(image, "/")
	name := image
	if slash >= 0 {
		name = image[slash+1:]
	}
	if i := strings.Index(name, ":"); i >= 0 {
		name = name[:i]
	}
	return name
}

func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
	jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
	if err != nil {
		return err
	}
	wanted := map[string]bool{}
	if job.NeedsSet {
		for _, n := range job.Needs {
			wanted[n] = true
		}
	} else {
		stageOrder := map[string]int{}
		n := 0
		for _, j := range jobs {
			if _, ok := stageOrder[j.Stage]; !ok {
				stageOrder[j.Stage] = n
				n++
			}
		}
		my := stageOrder[job.Stage]
		for _, j := range jobs {
			if stageOrder[j.Stage] < my {
				wanted[j.Name] = true
			}
		}
	}
	for _, j := range jobs {
		if !wanted[j.Name] || j.Status != "success" {
			continue
		}
		for _, rel := range j.ArtifactPaths {
			src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
			if err != nil {
				continue
			}
			dst := filepath.Join(work, filepath.FromSlash(rel))
			if err := copyFile(src, dst); err != nil {
				return err
			}
		}
	}
	return nil
}

func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
	var expires *time.Time
	if job.ArtifactExpire > 0 {
		t := time.Now().Add(job.ArtifactExpire)
		expires = &t
	}
	for _, rel := range job.ArtifactPaths {
		src := filepath.Join(work, filepath.FromSlash(rel))
		info, err := os.Stat(src)
		if err != nil || info.IsDir() {
			return fmt.Errorf("artifact %s ontbreekt", rel)
		}
		dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
		if err := copyFile(src, dst); err != nil {
			return err
		}
		if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
			return err
		}
	}
	return nil
}

func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
	name, _ := ci.SplitRef(p.Ref)
	refProtected, err := r.refIsProtected(ctx, repo.ID, p.Ref)
	if err != nil {
		return nil, nil, err
	}
	// Fork-MR's draaien op refs/heads/forks/...: code van buiten, dus
	// nooit groeps- of projectgeheimen en nooit als beschermde ref.
	forkRef := ci.IsForkBranch(name)
	if forkRef {
		refProtected = false
	}
	env := map[string]string{}
	var mask []string
	expand := map[string]bool{}
	if repo.OwnerKind == "group" && !forkRef {
		if flags, err := r.Store.SecretExpandFlags(ctx, "group", repo.OwnerID); err == nil {
			for k, on := range flags {
				expand[k] = on
			}
		}
		groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
		if err != nil {
			return nil, nil, err
		}
		for k, v := range groupSecrets {
			env[k] = v
			mask = append(mask, v)
		}
	}
	repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
	if err != nil {
		return nil, nil, err
	}
	if forkRef {
		repoSecrets = nil
	}
	repoFlags, _ := r.Store.SecretExpandFlags(ctx, "repo", repo.ID)
	for k, v := range repoSecrets {
		env[k] = v
		mask = append(mask, v)
		// Het project wint, ook voor de expand-vlag.
		expand[k] = repoFlags[k]
	}
	for k, v := range job.Variables {
		env[k] = v
		delete(expand, k)
	}
	base := strings.TrimRight(r.BaseURL, "/")
	dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
	var info *ci.MRInfo
	if p.MRID != 0 {
		if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
			target := ""
			if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
				target = strings.TrimSpace(string(out))
			}
			info = &ci.MRInfo{
				ID: m.ID, IID: m.Number, Title: m.Title,
				SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
				ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
				TargetSHA: target,
			}
		}
	}
	stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
	pre := ci.Variables(ci.Ref{
		SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
		DefaultBranch: repo.DefaultBranch, Source: p.Source,
		ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
		APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
	})
	for k, v := range pre {
		env[k] = v
	}
	env["CI_JOB_ID"] = fmt.Sprint(job.ID)
	env["CI_JOB_NAME"] = job.Name
	env["CI_JOB_STAGE"] = job.Stage
	env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
	env["CI_JOB_TOKEN"] = token
	if job.Environment != "" {
		env["CI_ENVIRONMENT_NAME"] = job.Environment
		if job.EnvironmentURL != "" {
			env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
		}
	}
	for k, v := range job.Variables {
		if strings.HasPrefix(k, "CI_NODE_") {
			env[k] = v
		}
	}
	// Variabelen met expand: na de volledige samenvoeging (groep, project,
	// job, CI_*). Alleen geheimen die in deze run bestaan doen mee, dus een
	// beschermd geheim lekt niet naar een onbeschermde ref.
	for k := range expand {
		if _, ok := env[k]; !ok || job.Variables[k] != "" {
			delete(expand, k)
		}
	}
	for k, v := range ci.ExpandVariables(env, expand) {
		env[k] = v
		if v != "" {
			mask = append(mask, v)
		}
	}
	// Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
	// en safe.directory er alsnog bij komt.
	markWorkspaceSafe(env)
	JobGitAuth(env, base+"/"+repo.FullPath(), token)
	mask = append(mask, token)
	return env, mask, nil
}

// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
// job-token. De header geldt alleen voor de URL van dit project (met en
// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
// het in elk image, zonder bestanden in de container.
func JobGitAuth(env map[string]string, projectURL, token string) {
	if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
		return
	}
	n := 0
	if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
		n = v
	}
	for _, u := range []string{projectURL, projectURL + ".git"} {
		env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
		env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
		n++
	}
	env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
}

// markWorkspaceSafe zet safe.directory voor elke job.
// De checkout is van de host-uid; de container-user is dat niet.
// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
// * of /workspace zet, blijft zoals hij is.
func markWorkspaceSafe(env map[string]string) {
	const key = "safe.directory"
	n := 0
	if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
		if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
			n = v
		}
	}
	for i := 0; i < n; i++ {
		if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
			continue
		}
		switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
		case "*", "/workspace":
			return
		}
	}
	env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
	env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
	env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
}

func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
	netName := fmt.Sprintf("bluegit-%d", job.ID)
	if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
		return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
	}
	var names []string
	cleanup := func() {
		cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		for _, name := range names {
			_, _ = execOut(cctx, r.Docker, "rm", "-f", name)
		}
		_, _ = execOut(cctx, r.Docker, "network", "rm", netName)
	}
	for i, svc := range job.Services {
		cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
		args := []string{"run", "-d", "--name", cname, "--network", netName}
		if svc.Alias != "" {
			args = append(args, "--network-alias", svc.Alias)
		}
		cmdEnv := append([]string{}, os.Environ()...)
		for k, v := range svc.Variables {
			args = append(args, "-e", k)
			cmdEnv = append(cmdEnv, k+"="+v)
		}
		if len(svc.Entrypoint) > 0 {
			args = append(args, "--entrypoint", svc.Entrypoint[0])
		}
		args = append(args, svc.Name)
		if len(svc.Entrypoint) > 1 {
			args = append(args, svc.Entrypoint[1:]...)
		}
		args = append(args, svc.Command...)
		cmd := exec.CommandContext(ctx, r.Docker, args...)
		cmd.Env = cmdEnv
		var stderr bytes.Buffer
		cmd.Stderr = &stderr
		if err := cmd.Run(); err != nil {
			cleanup()
			msg := strings.TrimSpace(stderr.String())
			if msg == "" {
				msg = err.Error()
			}
			return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
		}
		names = append(names, cname)
	}
	return netName, cleanup, nil
}

func shouldRetry(job store.Job, kind string) bool {
	if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
		return false
	}
	if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
		return true
	}
	if containsStr(job.RetryWhen, kind) {
		return true
	}
	return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
}

func classifyFailure(err error) string {
	msg := err.Error()
	if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
		return "runner_system_failure"
	}
	if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
		return "job_execution_timeout"
	}
	return "script_failure"
}

func containsStr(list []string, s string) bool {
	for _, item := range list {
		if item == s {
			return true
		}
	}
	return false
}

func safeSegment(name string) string {
	var b strings.Builder
	for _, r := range name {
		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
			b.WriteRune(r)
		} else {
			b.WriteByte('_')
		}
	}
	if b.Len() == 0 {
		return "job"
	}
	return b.String()
}

func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
	if len(job.CachePaths) == 0 {
		return nil
	}
	root := r.cacheDir(repoID, job.CacheKey)
	for _, rel := range job.CachePaths {
		src := filepath.Join(root, filepath.FromSlash(rel))
		if _, err := os.Stat(src); err != nil {
			continue
		}
		if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
			return err
		}
	}
	return nil
}

func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
	if len(job.CachePaths) == 0 {
		return nil
	}
	root := r.cacheDir(repoID, job.CacheKey)
	for _, rel := range job.CachePaths {
		src := filepath.Join(work, filepath.FromSlash(rel))
		info, err := os.Stat(src)
		if err != nil || info.IsDir() {
			continue
		}
		if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
			return err
		}
	}
	return nil
}

func (r *Runner) cacheDir(repoID int64, key string) string {
	if key == "" {
		key = "default"
	}
	var b strings.Builder
	for _, r := range key {
		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
			b.WriteRune(r)
		}
	}
	safe := b.String()
	if safe == "" {
		safe = "default"
	}
	return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
}

func stamp(line string) string {
	return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
}

type logSink struct {
	mu      sync.Mutex
	store   *store.Store
	jobID   int64
	out     func(string)
	secrets []string
	partial string
	current string
	opened  time.Time
}

func (l *logSink) Write(p []byte) (int, error) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.partial += string(p)
	for {
		i := strings.IndexByte(l.partial, '\n')
		if i < 0 {
			break
		}
		l.emit(strings.TrimRight(l.partial[:i], "\r"))
		l.partial = l.partial[i+1:]
	}
	return len(p), nil
}

func (l *logSink) flush() {
	l.mu.Lock()
	defer l.mu.Unlock()
	if strings.TrimSpace(l.partial) != "" {
		l.emit(l.partial)
	}
	l.partial = ""
}

func (l *logSink) emit(line string) {
	line = ci.Mask(line, l.secrets)
	if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
		if l.current != "" && strings.HasPrefix(line, "section_end:") {
			dur := int(time.Since(l.opened).Seconds())
			line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
			l.current = ""
		}
		if strings.HasPrefix(line, "section_start:") {
			l.current = line
			l.opened = time.Now()
		}
	}
	if l.out != nil {
		l.out(stamp(line))
		return
	}
	_ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
}

func (l *logSink) section(name, title string) {
	l.emit("section_start:" + name + ":" + title)
}

func (l *logSink) closeSection() {
	if l.current == "" {
		return
	}
	name := "prepare"
	if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
		name = parts[0]
	}
	l.emit("section_end:" + name + ":done")
}

func copyFile(src, dst string) error {
	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
		return err
	}
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	return err
}

func execOut(ctx context.Context, bin string, args ...string) (string, error) {
	if bin == "" {
		bin = "git"
	}
	cmd := exec.CommandContext(ctx, bin, args...)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		msg := strings.TrimSpace(stderr.String())
		if msg == "" {
			msg = err.Error()
		}
		return "", fmt.Errorf("%s", msg)
	}
	return stdout.String(), nil
}

// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
// werkmap en de bare repo kunnen andere eigenaars hebben, en met
// fs.protected_hardlinks geeft link() dan "Operation not permitted".
func cloneArgs(bare, work string) []string {
	return []string{"clone", "--no-hardlinks", bare, work}
}

// ProtectedRunning is het aantal lopende doNotInterrupt-jobs in dit proces.
func (r *Runner) ProtectedRunning() int64 { return r.protected.Load() }

// WaitProtected wacht bij het afsluiten tot alle lopende
// doNotInterrupt-jobs klaar zijn. Hun eigen timeout begrenst het wachten.
func (r *Runner) WaitProtected(log func(n int64)) {
	last := int64(-1)
	for n := r.protected.Load(); n > 0; n = r.protected.Load() {
		if n != last && log != nil {
			log(n)
			last = n
		}
		time.Sleep(time.Second)
	}
}
