wip runner module: store, routing, remote core
@@ -0,0 +1,353 @@
1
package runner
2
3
import (
4
"archive/tar"
5
"compress/gzip"
6
"context"
7
"errors"
8
"fmt"
9
"io"
10
"os"
11
"path/filepath"
12
"strings"
13
"sync"
14
"time"
15
16
"gitlab.com/Timfa/bluegit/internal/ci"
17
"gitlab.com/Timfa/bluegit/internal/store"
18
)
19
20
// RemotePayload is wat een aparte runner bij een job krijgt.
21
type RemotePayload struct {
22
JobID int64 `json:"job_id"`
23
PipelineID int64 `json:"pipeline_id"`
24
RepoID int64 `json:"repo_id"`
25
Job store.Job `json:"job"`
26
CloneURL string `json:"clone_url"`
27
SHA string `json:"sha"`
28
Script string `json:"script"`
29
Env map[string]string `json:"env"`
30
Mask []string `json:"mask"`
31
Timeout int64 `json:"timeout_seconds"`
32
}
33
34
var remoteMasks sync.Map // job id → []string
35
36
// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.
37
var RemoteOnlineGrace = 5 * time.Minute
38
39
// Claim zoekt een aangeboden job voor runner rn en zet hem op running.
40
// nil, nil als er niets is.
41
func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {
42
if r.storage() == "" {
43
return nil, nil
44
}
45
jobs, runs, err := r.Store.OfferedJobs(ctx)
46
if err != nil {
47
return nil, err
48
}
49
cache := map[int64][]Tier{}
50
for _, j := range jobs {
51
tiers := r.tiersFor(ctx, cache, j.RepoID)
52
if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {
53
continue
54
}
55
repo, err := r.Store.RepoByID(ctx, j.RepoID)
56
if err != nil || !r.mayRun(ctx, rn, repo) {
57
continue
58
}
59
p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)
60
if err != nil || (r.Hold != nil && r.Hold(p)) {
61
continue
62
}
63
if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
64
continue
65
}
66
ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)
67
if err != nil || !ok {
68
continue
69
}
70
pl, err := r.payload(ctx, repo, p, j, rn)
71
if err != nil {
72
r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())
73
continue
74
}
75
return pl, nil
76
}
77
return nil, nil
78
}
79
80
// mayRun is de privacycontrole: een account-runner krijgt alleen jobs
81
// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit
82
// repo's van die groep.
83
func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool {
84
switch rn.OwnerKind {
85
case "group":
86
return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID
87
case "user":
88
if repo.OwnerKind == "user" {
89
return repo.OwnerID == rn.OwnerID
90
}
91
role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)
92
return err == nil && role.CanRead()
93
}
94
return false
95
}
96
97
func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {
98
token, err := r.Store.CreateJobToken(ctx, job.ID)
99
if err != nil {
100
return nil, err
101
}
102
env, mask, err := r.environment(ctx, repo, p, job, token)
103
if err != nil {
104
return nil, err
105
}
106
remoteMasks.Store(job.ID, mask)
107
limit := r.Timeout
108
if job.Timeout > 0 {
109
limit = job.Timeout
110
}
111
script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})
112
return &RemotePayload{
113
JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,
114
CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",
115
SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,
116
Timeout: int64(limit / time.Second),
117
}, nil
118
}
119
120
// RemoteLog voegt logregels van een aparte runner toe. cancel is true als
121
// de job niet meer loopt (geannuleerd of afgebroken).
122
func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {
123
if job.Status != "running" {
124
return true, nil
125
}
126
if m, ok := remoteMasks.Load(job.ID); ok {
127
chunk = ci.Mask(chunk, m.([]string))
128
}
129
if len(chunk) > 0 {
130
err = r.Store.AppendJobLog(ctx, job.ID, chunk)
131
}
132
return false, err
133
}
134
135
// RemoteInputs zet de artifacts van eerdere jobs in dir.
136
func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {
137
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
138
if err != nil {
139
return err
140
}
141
return r.materialize(ctx, p, job, dir)
142
}
143
144
// RemoteArtifacts bewaart de geüploade artifacts uit dir.
145
func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {
146
return r.collect(ctx, job, dir)
147
}
148
149
// RemoteFinish rondt een job van een aparte runner af.
150
func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {
151
job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)
152
if err != nil {
153
return err
154
}
155
if job.Status != "running" {
156
return nil
157
}
158
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
159
if err != nil {
160
return err
161
}
162
var mask []string
163
if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {
164
mask = m.([]string)
165
}
166
extra, kind := "", ""
167
switch status {
168
case "success":
169
case "canceled":
170
extra = stamp("canceled")
171
default:
172
status = "failed"
173
if message == "" {
174
message = "job failed"
175
}
176
kind = classifyFailure(errors.New(message))
177
if strings.Contains(message, "job_execution_timeout") {
178
kind = "job_execution_timeout"
179
}
180
extra = stamp(ci.Mask(message, mask))
181
}
182
r.complete(p, job, p, status, extra, kind)
183
return nil
184
}
185
186
// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.
187
func (r *Runner) reapRemote(ctx context.Context) {
188
jobs, err := r.Store.RemoteRunningJobs(ctx)
189
if err != nil {
190
return
191
}
192
now := time.Now()
193
for _, rj := range jobs {
194
rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)
195
msg := ""
196
switch {
197
case err != nil || rn.Revoked:
198
msg = "runner_system_failure: runner was removed"
199
case now.Sub(rn.LastSeen) > RemoteOnlineGrace:
200
msg = "runner_system_failure: runner " + rn.Name + " stopped responding"
201
default:
202
limit := r.Timeout
203
if rj.Job.Timeout > 0 {
204
limit = rj.Job.Timeout
205
}
206
if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {
207
msg = "job_execution_timeout: no result from runner " + rn.Name
208
}
209
}
210
if msg != "" {
211
_ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)
212
}
213
}
214
}
215
216
// RunRemote draait een job op de runnermachine: clone, cache, inputs,
217
// container, cache opslaan. De werkmap blijft staan voor de artifacts.
218
// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.
219
func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
220
sink := &logSink{out: out, secrets: pl.Mask}
221
sink.section("prepare", "Preparing the docker executor")
222
_ = os.RemoveAll(work)
223
if err := os.MkdirAll(work, 0o755); err != nil {
224
sink.closeSection()
225
return err
226
}
227
if err := clone(ctx, work); err != nil {
228
sink.closeSection()
229
return fmt.Errorf("clone: %w", err)
230
}
231
if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {
232
sink.closeSection()
233
return fmt.Errorf("checkout %s: %w", pl.SHA, err)
234
}
235
if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {
236
sink.closeSection()
237
return err
238
}
239
if err := inputs(ctx, work); err != nil {
240
sink.closeSection()
241
return fmt.Errorf("artifacts: %w", err)
242
}
243
if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {
244
sink.closeSection()
245
return err
246
}
247
if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {
248
return err
249
}
250
for _, rel := range pl.Job.ArtifactPaths {
251
if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {
252
return fmt.Errorf("artifact %s ontbreekt", rel)
253
}
254
}
255
return r.saveCache(pl.RepoID, pl.Job, work)
256
}
257
258
// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.
259
// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.
260
func WriteTar(w io.Writer, dir string, rels []string) error {
261
gz := gzip.NewWriter(w)
262
tw := tar.NewWriter(gz)
263
add := func(rel string) error {
264
path := filepath.Join(dir, filepath.FromSlash(rel))
265
info, err := os.Stat(path)
266
if err != nil || !info.Mode().IsRegular() {
267
return nil
268
}
269
f, err := os.Open(path)
270
if err != nil {
271
return err
272
}
273
defer f.Close()
274
if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
275
return err
276
}
277
_, err = io.Copy(tw, f)
278
return err
279
}
280
if rels == nil {
281
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
282
if err != nil || info.IsDir() {
283
return err
284
}
285
rel, _ := filepath.Rel(dir, path)
286
return add(filepath.ToSlash(rel))
287
})
288
if err != nil {
289
return err
290
}
291
} else {
292
for _, rel := range rels {
293
if err := add(rel); err != nil {
294
return err
295
}
296
}
297
}
298
if err := tw.Close(); err != nil {
299
return err
300
}
301
return gz.Close()
302
}
303
304
// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;
305
// met allowed niet-nil alleen die paden. max begrenst de totale grootte.
306
func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {
307
gz, err := gzip.NewReader(rd)
308
if err != nil {
309
return err
310
}
311
tr := tar.NewReader(gz)
312
ok := map[string]bool{}
313
for _, a := range allowed {
314
ok[filepath.ToSlash(filepath.Clean(a))] = true
315
}
316
var total int64
317
for {
318
h, err := tr.Next()
319
if errors.Is(err, io.EOF) {
320
return nil
321
}
322
if err != nil {
323
return err
324
}
325
if h.Typeflag != tar.TypeReg {
326
continue
327
}
328
name := filepath.ToSlash(filepath.Clean(h.Name))
329
if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {
330
return fmt.Errorf("unsafe path %q", h.Name)
331
}
332
if allowed != nil && !ok[name] {
333
continue
334
}
335
total += h.Size
336
if max > 0 && total > max {
337
return errors.New("archive too large")
338
}
339
dst := filepath.Join(dir, filepath.FromSlash(name))
340
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
341
return err
342
}
343
f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
344
if err != nil {
345
return err
346
}
347
if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {
348
f.Close()
349
return err
350
}
351
f.Close()
352
}
353
}
@@ -0,0 +1,353 @@1
package runner2
3
import (4
"archive/tar"5
"compress/gzip"6
"context"7
"errors"8
"fmt"9
"io"10
"os"11
"path/filepath"12
"strings"13
"sync"14
"time"15
16
"gitlab.com/Timfa/bluegit/internal/ci"17
"gitlab.com/Timfa/bluegit/internal/store"18
)19
20
// RemotePayload is wat een aparte runner bij een job krijgt.21
type RemotePayload struct {22
JobID int64 `json:"job_id"`23
PipelineID int64 `json:"pipeline_id"`24
RepoID int64 `json:"repo_id"`25
Job store.Job `json:"job"`26
CloneURL string `json:"clone_url"`27
SHA string `json:"sha"`28
Script string `json:"script"`29
Env map[string]string `json:"env"`30
Mask []string `json:"mask"`31
Timeout int64 `json:"timeout_seconds"`32
}33
34
var remoteMasks sync.Map // job id → []string35
36
// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.37
var RemoteOnlineGrace = 5 * time.Minute38
39
// Claim zoekt een aangeboden job voor runner rn en zet hem op running.40
// nil, nil als er niets is.41
func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {42
if r.storage() == "" {43
return nil, nil44
}45
jobs, runs, err := r.Store.OfferedJobs(ctx)46
if err != nil {47
return nil, err48
}49
cache := map[int64][]Tier{}50
for _, j := range jobs {51
tiers := r.tiersFor(ctx, cache, j.RepoID)52
if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {53
continue54
}55
repo, err := r.Store.RepoByID(ctx, j.RepoID)56
if err != nil || !r.mayRun(ctx, rn, repo) {57
continue58
}59
p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)60
if err != nil || (r.Hold != nil && r.Hold(p)) {61
continue62
}63
if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {64
continue65
}66
ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)67
if err != nil || !ok {68
continue69
}70
pl, err := r.payload(ctx, repo, p, j, rn)71
if err != nil {72
r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())73
continue74
}75
return pl, nil76
}77
return nil, nil78
}79
80
// mayRun is de privacycontrole: een account-runner krijgt alleen jobs81
// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit82
// repo's van die groep.83
func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool {84
switch rn.OwnerKind {85
case "group":86
return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID87
case "user":88
if repo.OwnerKind == "user" {89
return repo.OwnerID == rn.OwnerID90
}91
role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)92
return err == nil && role.CanRead()93
}94
return false95
}96
97
func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {98
token, err := r.Store.CreateJobToken(ctx, job.ID)99
if err != nil {100
return nil, err101
}102
env, mask, err := r.environment(ctx, repo, p, job, token)103
if err != nil {104
return nil, err105
}106
remoteMasks.Store(job.ID, mask)107
limit := r.Timeout108
if job.Timeout > 0 {109
limit = job.Timeout110
}111
script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})112
return &RemotePayload{113
JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,114
CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",115
SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,116
Timeout: int64(limit / time.Second),117
}, nil118
}119
120
// RemoteLog voegt logregels van een aparte runner toe. cancel is true als121
// de job niet meer loopt (geannuleerd of afgebroken).122
func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {123
if job.Status != "running" {124
return true, nil125
}126
if m, ok := remoteMasks.Load(job.ID); ok {127
chunk = ci.Mask(chunk, m.([]string))128
}129
if len(chunk) > 0 {130
err = r.Store.AppendJobLog(ctx, job.ID, chunk)131
}132
return false, err133
}134
135
// RemoteInputs zet de artifacts van eerdere jobs in dir.136
func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {137
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)138
if err != nil {139
return err140
}141
return r.materialize(ctx, p, job, dir)142
}143
144
// RemoteArtifacts bewaart de geüploade artifacts uit dir.145
func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {146
return r.collect(ctx, job, dir)147
}148
149
// RemoteFinish rondt een job van een aparte runner af.150
func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {151
job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)152
if err != nil {153
return err154
}155
if job.Status != "running" {156
return nil157
}158
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)159
if err != nil {160
return err161
}162
var mask []string163
if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {164
mask = m.([]string)165
}166
extra, kind := "", ""167
switch status {168
case "success":169
case "canceled":170
extra = stamp("canceled")171
default:172
status = "failed"173
if message == "" {174
message = "job failed"175
}176
kind = classifyFailure(errors.New(message))177
if strings.Contains(message, "job_execution_timeout") {178
kind = "job_execution_timeout"179
}180
extra = stamp(ci.Mask(message, mask))181
}182
r.complete(p, job, p, status, extra, kind)183
return nil184
}185
186
// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.187
func (r *Runner) reapRemote(ctx context.Context) {188
jobs, err := r.Store.RemoteRunningJobs(ctx)189
if err != nil {190
return191
}192
now := time.Now()193
for _, rj := range jobs {194
rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)195
msg := ""196
switch {197
case err != nil || rn.Revoked:198
msg = "runner_system_failure: runner was removed"199
case now.Sub(rn.LastSeen) > RemoteOnlineGrace:200
msg = "runner_system_failure: runner " + rn.Name + " stopped responding"201
default:202
limit := r.Timeout203
if rj.Job.Timeout > 0 {204
limit = rj.Job.Timeout205
}206
if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {207
msg = "job_execution_timeout: no result from runner " + rn.Name208
}209
}210
if msg != "" {211
_ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)212
}213
}214
}215
216
// RunRemote draait een job op de runnermachine: clone, cache, inputs,217
// container, cache opslaan. De werkmap blijft staan voor de artifacts.218
// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.219
func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {220
sink := &logSink{out: out, secrets: pl.Mask}221
sink.section("prepare", "Preparing the docker executor")222
_ = os.RemoveAll(work)223
if err := os.MkdirAll(work, 0o755); err != nil {224
sink.closeSection()225
return err226
}227
if err := clone(ctx, work); err != nil {228
sink.closeSection()229
return fmt.Errorf("clone: %w", err)230
}231
if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {232
sink.closeSection()233
return fmt.Errorf("checkout %s: %w", pl.SHA, err)234
}235
if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {236
sink.closeSection()237
return err238
}239
if err := inputs(ctx, work); err != nil {240
sink.closeSection()241
return fmt.Errorf("artifacts: %w", err)242
}243
if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {244
sink.closeSection()245
return err246
}247
if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {248
return err249
}250
for _, rel := range pl.Job.ArtifactPaths {251
if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {252
return fmt.Errorf("artifact %s ontbreekt", rel)253
}254
}255
return r.saveCache(pl.RepoID, pl.Job, work)256
}257
258
// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.259
// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.260
func WriteTar(w io.Writer, dir string, rels []string) error {261
gz := gzip.NewWriter(w)262
tw := tar.NewWriter(gz)263
add := func(rel string) error {264
path := filepath.Join(dir, filepath.FromSlash(rel))265
info, err := os.Stat(path)266
if err != nil || !info.Mode().IsRegular() {267
return nil268
}269
f, err := os.Open(path)270
if err != nil {271
return err272
}273
defer f.Close()274
if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {275
return err276
}277
_, err = io.Copy(tw, f)278
return err279
}280
if rels == nil {281
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {282
if err != nil || info.IsDir() {283
return err284
}285
rel, _ := filepath.Rel(dir, path)286
return add(filepath.ToSlash(rel))287
})288
if err != nil {289
return err290
}291
} else {292
for _, rel := range rels {293
if err := add(rel); err != nil {294
return err295
}296
}297
}298
if err := tw.Close(); err != nil {299
return err300
}301
return gz.Close()302
}303
304
// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;305
// met allowed niet-nil alleen die paden. max begrenst de totale grootte.306
func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {307
gz, err := gzip.NewReader(rd)308
if err != nil {309
return err310
}311
tr := tar.NewReader(gz)312
ok := map[string]bool{}313
for _, a := range allowed {314
ok[filepath.ToSlash(filepath.Clean(a))] = true315
}316
var total int64317
for {318
h, err := tr.Next()319
if errors.Is(err, io.EOF) {320
return nil321
}322
if err != nil {323
return err324
}325
if h.Typeflag != tar.TypeReg {326
continue327
}328
name := filepath.ToSlash(filepath.Clean(h.Name))329
if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {330
return fmt.Errorf("unsafe path %q", h.Name)331
}332
if allowed != nil && !ok[name] {333
continue334
}335
total += h.Size336
if max > 0 && total > max {337
return errors.New("archive too large")338
}339
dst := filepath.Join(dir, filepath.FromSlash(name))340
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {341
return err342
}343
f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)344
if err != nil {345
return err346
}347
if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {348
f.Close()349
return err350
}351
f.Close()352
}353
}@@ -0,0 +1,149 @@
1
package runner
2
3
import (
4
"context"
5
"time"
6
7
"gitlab.com/Timfa/bluegit/internal/access"
8
"gitlab.com/Timfa/bluegit/internal/store"
9
)
10
11
// Tier is één trede in de runnervolgorde van een job: de ingebouwde
12
// BlueGit-runner of een set aparte runners.
13
type Tier struct {
14
Builtin bool
15
Runners []store.Runner
16
}
17
18
// OfferTimeout: zo lang krijgen online aparte runners in een trede om een
19
// job op te pakken voordat de volgende trede aan de beurt is.
20
var OfferTimeout = 20 * time.Second
21
22
// Order berekent de volgorde puur uit de gegevens; zie Tiers.
23
//
24
// Accountrepo: eigen runners eerst (own_first) of de BlueGit-runner eerst
25
// en eigen runners alleen als die niet meteen vrij is (builtin_first).
26
// Groepsrepo: groepsrunners → account-runners van maintainers die
27
// toestemming gaven → BlueGit-runner; met builtinFirst staat de
28
// BlueGit-runner vooraan. Lege treden vallen weg.
29
func Order(ownerKind, userPref string, builtinFirst bool, own, group, optedIn []store.Runner) []Tier {
30
var tiers []Tier
31
add := func(t Tier) {
32
if t.Builtin || len(t.Runners) > 0 {
33
tiers = append(tiers, t)
34
}
35
}
36
builtin := Tier{Builtin: true}
37
if ownerKind == "group" {
38
if builtinFirst {
39
add(builtin)
40
}
41
add(Tier{Runners: group})
42
add(Tier{Runners: optedIn})
43
if !builtinFirst {
44
add(builtin)
45
}
46
return tiers
47
}
48
if userPref == store.RunnerPrefOwnFirst {
49
add(Tier{Runners: own})
50
add(builtin)
51
} else {
52
add(builtin)
53
add(Tier{Runners: own})
54
}
55
return tiers
56
}
57
58
// Tiers geeft de runnervolgorde voor een repo.
59
func Tiers(ctx context.Context, st *store.Store, repo store.Repo) ([]Tier, error) {
60
if repo.OwnerKind == "group" {
61
group, err := st.ListRunners(ctx, "group", repo.OwnerID)
62
if err != nil {
63
return nil, err
64
}
65
users, err := st.RunnerOptIns(ctx, repo.OwnerID)
66
if err != nil {
67
return nil, err
68
}
69
var opted []store.Runner
70
for _, u := range users {
71
// Toestemming geldt alleen zolang de gebruiker maintainer is.
72
role, err := st.GroupRole(ctx, repo.OwnerID, u)
73
if err != nil || !role.AtLeast(access.RoleMaintainer) {
74
continue
75
}
76
rs, err := st.ListRunners(ctx, "user", u)
77
if err != nil {
78
return nil, err
79
}
80
opted = append(opted, rs...)
81
}
82
return Order("group", "", st.GroupBuiltinFirst(ctx, repo.OwnerID), nil, group, opted), nil
83
}
84
own, err := st.ListRunners(ctx, "user", repo.OwnerID)
85
if err != nil {
86
return nil, err
87
}
88
return Order("user", st.UserRunnerPref(ctx, repo.OwnerID), false, own, nil, nil), nil
89
}
90
91
func anyOnline(rs []store.Runner, now time.Time) bool {
92
for _, r := range rs {
93
if r.Online(now) {
94
return true
95
}
96
}
97
return false
98
}
99
100
// Step is de beslissing voor een wachtende job in deze tick.
101
type Step struct {
102
Tier int // trede waar de job nu staat
103
Builtin bool // nu ingebouwd starten
104
Moved bool // aanbod is verplaatst; opslaan
105
}
106
107
// Advance kiest wat er met een wachtende job gebeurt. builtinFree zegt of
108
// de ingebouwde runner nu een plek heeft. Elke trede wordt hooguit één
109
// keer per tick bekeken.
110
func Advance(tiers []Tier, tier int, offerAt, now time.Time, builtinFree bool) Step {
111
n := len(tiers)
112
if n == 0 {
113
return Step{}
114
}
115
if tier < 0 || tier >= n {
116
tier = 0
117
}
118
moved := offerAt.IsZero()
119
if moved {
120
offerAt = now
121
}
122
for i := 0; i < n; i++ {
123
t := tiers[tier]
124
if t.Builtin {
125
if builtinFree {
126
return Step{Tier: tier, Builtin: true, Moved: moved}
127
}
128
} else if anyOnline(t.Runners, now) && (now.Sub(offerAt) < OfferTimeout || i > 0 && moved) {
129
return Step{Tier: tier, Moved: moved}
130
}
131
tier = (tier + 1) % n
132
offerAt = now
133
moved = true
134
}
135
return Step{Tier: tier, Moved: moved}
136
}
137
138
// RunnerInTier zegt of runner id in de trede staat.
139
func RunnerInTier(tiers []Tier, tier int, id int64) bool {
140
if tier < 0 || tier >= len(tiers) {
141
return false
142
}
143
for _, r := range tiers[tier].Runners {
144
if r.ID == id {
145
return true
146
}
147
}
148
return false
149
}
@@ -0,0 +1,149 @@1
package runner2
3
import (4
"context"5
"time"6
7
"gitlab.com/Timfa/bluegit/internal/access"8
"gitlab.com/Timfa/bluegit/internal/store"9
)10
11
// Tier is één trede in de runnervolgorde van een job: de ingebouwde12
// BlueGit-runner of een set aparte runners.13
type Tier struct {14
Builtin bool15
Runners []store.Runner16
}17
18
// OfferTimeout: zo lang krijgen online aparte runners in een trede om een19
// job op te pakken voordat de volgende trede aan de beurt is.20
var OfferTimeout = 20 * time.Second21
22
// Order berekent de volgorde puur uit de gegevens; zie Tiers.23
//24
// Accountrepo: eigen runners eerst (own_first) of de BlueGit-runner eerst25
// en eigen runners alleen als die niet meteen vrij is (builtin_first).26
// Groepsrepo: groepsrunners → account-runners van maintainers die27
// toestemming gaven → BlueGit-runner; met builtinFirst staat de28
// BlueGit-runner vooraan. Lege treden vallen weg.29
func Order(ownerKind, userPref string, builtinFirst bool, own, group, optedIn []store.Runner) []Tier {30
var tiers []Tier31
add := func(t Tier) {32
if t.Builtin || len(t.Runners) > 0 {33
tiers = append(tiers, t)34
}35
}36
builtin := Tier{Builtin: true}37
if ownerKind == "group" {38
if builtinFirst {39
add(builtin)40
}41
add(Tier{Runners: group})42
add(Tier{Runners: optedIn})43
if !builtinFirst {44
add(builtin)45
}46
return tiers47
}48
if userPref == store.RunnerPrefOwnFirst {49
add(Tier{Runners: own})50
add(builtin)51
} else {52
add(builtin)53
add(Tier{Runners: own})54
}55
return tiers56
}57
58
// Tiers geeft de runnervolgorde voor een repo.59
func Tiers(ctx context.Context, st *store.Store, repo store.Repo) ([]Tier, error) {60
if repo.OwnerKind == "group" {61
group, err := st.ListRunners(ctx, "group", repo.OwnerID)62
if err != nil {63
return nil, err64
}65
users, err := st.RunnerOptIns(ctx, repo.OwnerID)66
if err != nil {67
return nil, err68
}69
var opted []store.Runner70
for _, u := range users {71
// Toestemming geldt alleen zolang de gebruiker maintainer is.72
role, err := st.GroupRole(ctx, repo.OwnerID, u)73
if err != nil || !role.AtLeast(access.RoleMaintainer) {74
continue75
}76
rs, err := st.ListRunners(ctx, "user", u)77
if err != nil {78
return nil, err79
}80
opted = append(opted, rs...)81
}82
return Order("group", "", st.GroupBuiltinFirst(ctx, repo.OwnerID), nil, group, opted), nil83
}84
own, err := st.ListRunners(ctx, "user", repo.OwnerID)85
if err != nil {86
return nil, err87
}88
return Order("user", st.UserRunnerPref(ctx, repo.OwnerID), false, own, nil, nil), nil89
}90
91
func anyOnline(rs []store.Runner, now time.Time) bool {92
for _, r := range rs {93
if r.Online(now) {94
return true95
}96
}97
return false98
}99
100
// Step is de beslissing voor een wachtende job in deze tick.101
type Step struct {102
Tier int // trede waar de job nu staat103
Builtin bool // nu ingebouwd starten104
Moved bool // aanbod is verplaatst; opslaan105
}106
107
// Advance kiest wat er met een wachtende job gebeurt. builtinFree zegt of108
// de ingebouwde runner nu een plek heeft. Elke trede wordt hooguit één109
// keer per tick bekeken.110
func Advance(tiers []Tier, tier int, offerAt, now time.Time, builtinFree bool) Step {111
n := len(tiers)112
if n == 0 {113
return Step{}114
}115
if tier < 0 || tier >= n {116
tier = 0117
}118
moved := offerAt.IsZero()119
if moved {120
offerAt = now121
}122
for i := 0; i < n; i++ {123
t := tiers[tier]124
if t.Builtin {125
if builtinFree {126
return Step{Tier: tier, Builtin: true, Moved: moved}127
}128
} else if anyOnline(t.Runners, now) && (now.Sub(offerAt) < OfferTimeout || i > 0 && moved) {129
return Step{Tier: tier, Moved: moved}130
}131
tier = (tier + 1) % n132
offerAt = now133
moved = true134
}135
return Step{Tier: tier, Moved: moved}136
}137
138
// RunnerInTier zegt of runner id in de trede staat.139
func RunnerInTier(tiers []Tier, tier int, id int64) bool {140
if tier < 0 || tier >= len(tiers) {141
return false142
}143
for _, r := range tiers[tier].Runners {144
if r.ID == id {145
return true146
}147
}148
return false149
}@@ -117,6 +117,7 @@ func (r *Runner) Tick(ctx context.Context) {
117
117
for _, p := range pipes {
118
118
r.schedule(ctx, p)
119
119
}
120
r.reapRemote(ctx)
120
121
}
121
122
122
123
func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
@@ -135,6 +136,7 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
135
136
specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
136
137
status[j.Name] = j.Status
137
138
}
139
tierCache := map[int64][]Tier{}
138
140
for _, j := range jobs {
139
141
if j.Status != "queued" && j.Status != "manual" {
140
142
continue
@@ -155,7 +157,21 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
155
157
}
156
158
// Meer jobs dan ci.concurrency blijven queued, met een plek
157
159
// in de wachtrij. Het slot gaat pas vrij als execute klaar is.
158
if !r.acquireSlot() {
160
free := r.acquireSlot()
161
if tiers := r.tiersFor(ctx, tierCache, j.RepoID); len(tiers) > 1 {
162
jr := r.Store.JobRun(ctx, j.ID)
163
step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
164
if !step.Builtin {
165
if free {
166
r.releaseSlot()
167
}
168
if step.Moved {
169
_ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
170
}
171
continue
172
}
173
}
174
if !free {
159
175
continue
160
176
}
161
177
ok, err := r.Store.MarkJobRunning(ctx, j.ID)
@@ -163,6 +179,8 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
163
179
r.releaseSlot()
164
180
continue
165
181
}
182
_ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
183
_ = r.Store.ClearJobOffer(ctx, j.ID)
166
184
status[j.Name] = "running"
167
185
job := j
168
186
pipe := p
@@ -217,6 +235,26 @@ func (r *Runner) settle(ctx context.Context, pipelineID int64) {
217
235
}
218
236
}
219
237
238
// BuiltinName is de naam van de ingebouwde runner in de job-UI.
239
func (r *Runner) BuiltinName() string {
240
if r.Name != "" {
241
return r.Name
242
}
243
return "BlueGit runner"
244
}
245
246
func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64) []Tier {
247
if t, ok := cache[repoID]; ok {
248
return t
249
}
250
var tiers []Tier
251
if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
252
tiers, _ = Tiers(ctx, r.Store, repo)
253
}
254
cache[repoID] = tiers
255
return tiers
256
}
257
220
258
func indexOf(jobs []store.Job, name string) int {
221
259
for i, j := range jobs {
222
260
if j.Name == name {
@@ -285,8 +323,15 @@ func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
285
323
kind = classifyFailure(err)
286
324
extra = stamp(ci.Mask(err.Error(), mask))
287
325
}
326
r.complete(p, job, before, status, extra, kind)
327
}
328
329
// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
330
// en webhook. Gedeeld door de ingebouwde en de aparte runner.
331
func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
288
332
if status == "failed" && shouldRetry(job, kind) {
289
333
note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
334
_ = r.Store.ClearJobOffer(context.Background(), job.ID)
290
335
if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
291
336
r.Log.Error("job opnieuw", "err", ferr)
292
337
} else if ferr == nil {
@@ -384,15 +429,31 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
384
429
return nil, err
385
430
}
386
431
sink.secrets = mask
432
if err := r.execContainer(ctx, sink, job, work, env); err != nil {
433
return mask, err
434
}
435
if err := r.collect(ctx, job, work); err != nil {
436
return mask, err
437
}
438
if err := r.saveCache(repo.ID, job, work); err != nil {
439
return mask, err
440
}
441
return mask, nil
442
}
443
444
// execContainer haalt het image op, start services en draait het
445
// jobscript in de container. De sectie prepare moet open staan.
446
func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
387
447
ref, lines, rerr := r.resolveImage(ctx, job.Image)
388
448
for _, line := range lines {
389
449
sink.emit(line)
390
450
}
391
451
if rerr != nil {
392
452
sink.closeSection()
393
return mask, rerr
453
return rerr
394
454
}
395
455
job.Image = ref
456
job.Services = append([]store.Service(nil), job.Services...)
396
457
for i := range job.Services {
397
458
sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
398
459
for _, line := range slines {
@@ -400,16 +461,19 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
400
461
}
401
462
if serr != nil {
402
463
sink.closeSection()
403
return mask, serr
464
return serr
404
465
}
405
466
job.Services[i].Name = sref
406
467
}
407
468
network := r.Network
469
if network == "" {
470
network = "bridge"
471
}
408
472
if len(job.Services) > 0 {
409
473
net, cleanup, serr := r.startServices(ctx, job)
410
474
if serr != nil {
411
475
sink.closeSection()
412
return mask, serr
476
return serr
413
477
}
414
478
defer cleanup()
415
479
network = net
@@ -429,18 +493,12 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
429
493
cmd.Env = cmdEnv
430
494
cmd.Stdout = sink
431
495
cmd.Stderr = sink
432
err = cmd.Run()
496
err := cmd.Run()
433
497
sink.flush()
434
498
if err != nil {
435
return mask, fmt.Errorf("container ended with an error: %w", err)
499
return fmt.Errorf("container ended with an error: %w", err)
436
500
}
437
if err := r.collect(ctx, job, work); err != nil {
438
return mask, err
439
}
440
if err := r.saveCache(repo.ID, job, work); err != nil {
441
return mask, err
442
}
443
return mask, nil
501
return nil
444
502
}
445
503
446
504
// placeNote zijn de vaste regels in de sectie Preparing environment.
@@ -985,6 +1043,7 @@ type logSink struct {
985
1043
mu sync.Mutex
986
1044
store *store.Store
987
1045
jobID int64
1046
out func(string)
988
1047
secrets []string
989
1048
partial string
990
1049
current string
@@ -1028,6 +1087,10 @@ func (l *logSink) emit(line string) {
1028
1087
l.opened = time.Now()
1029
1088
}
1030
1089
}
1090
if l.out != nil {
1091
l.out(stamp(line))
1092
return
1093
}
1031
1094
_ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1032
1095
}
1033
1096
@@ -117,6 +117,7 @@ func (r *Runner) Tick(ctx context.Context) {117
for _, p := range pipes {117
for _, p := range pipes {118
r.schedule(ctx, p)118
r.schedule(ctx, p)119
}119
}120
r.reapRemote(ctx)120
}121
}121
122
122
func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {123
func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {@@ -135,6 +136,7 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {135
specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}136
specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}136
status[j.Name] = j.Status137
status[j.Name] = j.Status137
}138
}139
tierCache := map[int64][]Tier{}138
for _, j := range jobs {140
for _, j := range jobs {139
if j.Status != "queued" && j.Status != "manual" {141
if j.Status != "queued" && j.Status != "manual" {140
continue142
continue@@ -155,7 +157,21 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {155
}157
}156
// Meer jobs dan ci.concurrency blijven queued, met een plek158
// Meer jobs dan ci.concurrency blijven queued, met een plek157
// in de wachtrij. Het slot gaat pas vrij als execute klaar is.159
// in de wachtrij. Het slot gaat pas vrij als execute klaar is.158
if !r.acquireSlot() {160
free := r.acquireSlot()161
if tiers := r.tiersFor(ctx, tierCache, j.RepoID); len(tiers) > 1 {162
jr := r.Store.JobRun(ctx, j.ID)163
step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)164
if !step.Builtin {165
if free {166
r.releaseSlot()167
}168
if step.Moved {169
_ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())170
}171
continue172
}173
}174
if !free {159
continue175
continue160
}176
}161
ok, err := r.Store.MarkJobRunning(ctx, j.ID)177
ok, err := r.Store.MarkJobRunning(ctx, j.ID)@@ -163,6 +179,8 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {163
r.releaseSlot()179
r.releaseSlot()164
continue180
continue165
}181
}182
_ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())183
_ = r.Store.ClearJobOffer(ctx, j.ID)166
status[j.Name] = "running"184
status[j.Name] = "running"167
job := j185
job := j168
pipe := p186
pipe := p@@ -217,6 +235,26 @@ func (r *Runner) settle(ctx context.Context, pipelineID int64) {217
}235
}218
}236
}219
237
238
// BuiltinName is de naam van de ingebouwde runner in de job-UI.239
func (r *Runner) BuiltinName() string {240
if r.Name != "" {241
return r.Name242
}243
return "BlueGit runner"244
}245
246
func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64) []Tier {247
if t, ok := cache[repoID]; ok {248
return t249
}250
var tiers []Tier251
if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {252
tiers, _ = Tiers(ctx, r.Store, repo)253
}254
cache[repoID] = tiers255
return tiers256
}257
220
func indexOf(jobs []store.Job, name string) int {258
func indexOf(jobs []store.Job, name string) int {221
for i, j := range jobs {259
for i, j := range jobs {222
if j.Name == name {260
if j.Name == name {@@ -285,8 +323,15 @@ func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {285
kind = classifyFailure(err)323
kind = classifyFailure(err)286
extra = stamp(ci.Mask(err.Error(), mask))324
extra = stamp(ci.Mask(err.Error(), mask))287
}325
}326
r.complete(p, job, before, status, extra, kind)327
}328
329
// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus330
// en webhook. Gedeeld door de ingebouwde en de aparte runner.331
func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {288
if status == "failed" && shouldRetry(job, kind) {332
if status == "failed" && shouldRetry(job, kind) {289
note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))333
note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))334
_ = r.Store.ClearJobOffer(context.Background(), job.ID)290
if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {335
if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {291
r.Log.Error("job opnieuw", "err", ferr)336
r.Log.Error("job opnieuw", "err", ferr)292
} else if ferr == nil {337
} else if ferr == nil {@@ -384,15 +429,31 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([384
return nil, err429
return nil, err385
}430
}386
sink.secrets = mask431
sink.secrets = mask432
if err := r.execContainer(ctx, sink, job, work, env); err != nil {433
return mask, err434
}435
if err := r.collect(ctx, job, work); err != nil {436
return mask, err437
}438
if err := r.saveCache(repo.ID, job, work); err != nil {439
return mask, err440
}441
return mask, nil442
}443
444
// execContainer haalt het image op, start services en draait het445
// jobscript in de container. De sectie prepare moet open staan.446
func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {387
ref, lines, rerr := r.resolveImage(ctx, job.Image)447
ref, lines, rerr := r.resolveImage(ctx, job.Image)388
for _, line := range lines {448
for _, line := range lines {389
sink.emit(line)449
sink.emit(line)390
}450
}391
if rerr != nil {451
if rerr != nil {392
sink.closeSection()452
sink.closeSection()393
return mask, rerr453
return rerr394
}454
}395
job.Image = ref455
job.Image = ref456
job.Services = append([]store.Service(nil), job.Services...)396
for i := range job.Services {457
for i := range job.Services {397
sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)458
sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)398
for _, line := range slines {459
for _, line := range slines {@@ -400,16 +461,19 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([400
}461
}401
if serr != nil {462
if serr != nil {402
sink.closeSection()463
sink.closeSection()403
return mask, serr464
return serr404
}465
}405
job.Services[i].Name = sref466
job.Services[i].Name = sref406
}467
}407
network := r.Network468
network := r.Network469
if network == "" {470
network = "bridge"471
}408
if len(job.Services) > 0 {472
if len(job.Services) > 0 {409
net, cleanup, serr := r.startServices(ctx, job)473
net, cleanup, serr := r.startServices(ctx, job)410
if serr != nil {474
if serr != nil {411
sink.closeSection()475
sink.closeSection()412
return mask, serr476
return serr413
}477
}414
defer cleanup()478
defer cleanup()415
network = net479
network = net@@ -429,18 +493,12 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([429
cmd.Env = cmdEnv493
cmd.Env = cmdEnv430
cmd.Stdout = sink494
cmd.Stdout = sink431
cmd.Stderr = sink495
cmd.Stderr = sink432
err = cmd.Run()496
err := cmd.Run()433
sink.flush()497
sink.flush()434
if err != nil {498
if err != nil {435
return mask, fmt.Errorf("container ended with an error: %w", err)499
return fmt.Errorf("container ended with an error: %w", err)436
}500
}437
if err := r.collect(ctx, job, work); err != nil {501
return nil438
return mask, err439
}440
if err := r.saveCache(repo.ID, job, work); err != nil {441
return mask, err442
}443
return mask, nil444
}502
}445
503
446
// placeNote zijn de vaste regels in de sectie Preparing environment.504
// placeNote zijn de vaste regels in de sectie Preparing environment.@@ -985,6 +1043,7 @@ type logSink struct {985
mu sync.Mutex1043
mu sync.Mutex986
store *store.Store1044
store *store.Store987
jobID int641045
jobID int641046
out func(string)988
secrets []string1047
secrets []string989
partial string1048
partial string990
current string1049
current string@@ -1028,6 +1087,10 @@ func (l *logSink) emit(line string) {1028
l.opened = time.Now()1087
l.opened = time.Now()1029
}1088
}1030
}1089
}1090
if l.out != nil {1091
l.out(stamp(line))1092
return1093
}1031
_ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))1094
_ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))1032
}1095
}1033
1096
@@ -0,0 +1,352 @@
1
package store
2
3
import (
4
"context"
5
"database/sql"
6
"errors"
7
"strings"
8
"time"
9
)
10
11
// Runner is een aparte bluegit-runner op een eigen machine, gekoppeld aan
12
// een account (owner_kind user) of een groep (owner_kind group).
13
type Runner struct {
14
ID int64
15
OwnerKind string
16
OwnerID int64
17
Name string
18
CreatedAt time.Time
19
LastSeen time.Time
20
Version string
21
Revoked bool
22
}
23
24
// RunnerOnlineWindow: zo lang na de laatste heartbeat telt een runner als online.
25
const RunnerOnlineWindow = 90 * time.Second
26
27
// Online zegt of de runner recent contact had.
28
func (r Runner) Online(now time.Time) bool {
29
return !r.Revoked && !r.LastSeen.IsZero() && now.Sub(r.LastSeen) < RunnerOnlineWindow
30
}
31
32
// Runner-voorkeuren van een account.
33
const (
34
RunnerPrefOwnFirst = "own_first"
35
RunnerPrefBuiltinFirst = "builtin_first"
36
)
37
38
// ErrRegToken is een onbekend, gebruikt of verlopen registratietoken.
39
var ErrRegToken = errors.New("registration token invalid or expired")
40
41
// CreateRunnerRegToken maakt een eenmalig registratietoken (24 uur geldig).
42
func (s *Store) CreateRunnerRegToken(ctx context.Context, ownerKind string, ownerID, by int64) (string, error) {
43
plain, err := newToken("bgrr_")
44
if err != nil {
45
return "", err
46
}
47
exp := time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)
48
_, err = s.db.ExecContext(ctx, `INSERT INTO runner_reg_tokens (owner_kind, owner_id, token_hash, created_by, expires_at) VALUES (?, ?, ?, ?, ?)`,
49
ownerKind, ownerID, hashToken(plain), by, exp)
50
return plain, err
51
}
52
53
// RegisterRunner wisselt een registratietoken in voor een runner en zijn
54
// eigen runner-token (bgr_). Alleen de hash wordt bewaard.
55
func (s *Store) RegisterRunner(ctx context.Context, regToken, name, version string) (Runner, string, error) {
56
tx, err := s.db.BeginTx(ctx, nil)
57
if err != nil {
58
return Runner{}, "", err
59
}
60
defer func() { _ = tx.Rollback() }()
61
var id, ownerID, by int64
62
var kind, exp string
63
err = tx.QueryRowContext(ctx, `SELECT id, owner_kind, owner_id, created_by, expires_at FROM runner_reg_tokens WHERE token_hash = ? AND used = 0`, hashToken(regToken)).Scan(&id, &kind, &ownerID, &by, &exp)
64
if err != nil || parseTime(exp).Before(time.Now()) {
65
return Runner{}, "", ErrRegToken
66
}
67
if _, err := tx.ExecContext(ctx, `UPDATE runner_reg_tokens SET used = 1 WHERE id = ?`, id); err != nil {
68
return Runner{}, "", err
69
}
70
plain, err := newToken("bgr_")
71
if err != nil {
72
return Runner{}, "", err
73
}
74
name = strings.TrimSpace(name)
75
if name == "" {
76
name = "runner"
77
}
78
if len(name) > 80 {
79
name = name[:80]
80
}
81
at := now()
82
res, err := tx.ExecContext(ctx, `INSERT INTO runners (owner_kind, owner_id, name, token_hash, created_by, created_at, last_seen, version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
83
kind, ownerID, name, hashToken(plain), by, at, at, version)
84
if err != nil {
85
return Runner{}, "", err
86
}
87
rid, _ := res.LastInsertId()
88
if err := tx.Commit(); err != nil {
89
return Runner{}, "", err
90
}
91
return Runner{ID: rid, OwnerKind: kind, OwnerID: ownerID, Name: name, CreatedAt: parseTime(at), LastSeen: parseTime(at), Version: version}, plain, nil
92
}
93
94
const runnerCols = `id, owner_kind, owner_id, name, created_at, last_seen, version, revoked`
95
96
func scanRunner(sc interface{ Scan(...any) error }) (Runner, error) {
97
var r Runner
98
var created, seen string
99
var revoked int
100
if err := sc.Scan(&r.ID, &r.OwnerKind, &r.OwnerID, &r.Name, &created, &seen, &r.Version, &revoked); err != nil {
101
return Runner{}, err
102
}
103
r.CreatedAt, r.LastSeen, r.Revoked = parseTime(created), parseTime(seen), revoked == 1
104
return r, nil
105
}
106
107
// RunnerByToken geeft de runner bij een bgr_-token. Ingetrokken telt niet.
108
func (s *Store) RunnerByToken(ctx context.Context, token string) (Runner, error) {
109
if !strings.HasPrefix(token, "bgr_") {
110
return Runner{}, errNotFound
111
}
112
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE token_hash = ? AND revoked = 0`, hashToken(token)))
113
if errors.Is(err, sql.ErrNoRows) {
114
return Runner{}, errNotFound
115
}
116
return r, err
117
}
118
119
// RunnerByID geeft één runner.
120
func (s *Store) RunnerByID(ctx context.Context, id int64) (Runner, error) {
121
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE id = ?`, id))
122
if errors.Is(err, sql.ErrNoRows) {
123
return Runner{}, errNotFound
124
}
125
return r, err
126
}
127
128
// TouchRunner is de heartbeat.
129
func (s *Store) TouchRunner(ctx context.Context, id int64, version string) error {
130
_, err := s.db.ExecContext(ctx, `UPDATE runners SET last_seen = ?, version = CASE WHEN ? = '' THEN version ELSE ? END WHERE id = ?`, now(), version, version, id)
131
return err
132
}
133
134
// ListRunners geeft de niet-ingetrokken runners van een eigenaar.
135
func (s *Store) ListRunners(ctx context.Context, ownerKind string, ownerID int64) ([]Runner, error) {
136
rows, err := s.db.QueryContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE owner_kind = ? AND owner_id = ? AND revoked = 0 ORDER BY id`, ownerKind, ownerID)
137
if err != nil {
138
return nil, err
139
}
140
defer rows.Close()
141
var out []Runner
142
for rows.Next() {
143
r, err := scanRunner(rows)
144
if err != nil {
145
return nil, err
146
}
147
out = append(out, r)
148
}
149
return out, rows.Err()
150
}
151
152
// RevokeRunner trekt een runner in; zijn token werkt daarna niet meer.
153
func (s *Store) RevokeRunner(ctx context.Context, ownerKind string, ownerID, id int64) error {
154
res, err := s.db.ExecContext(ctx, `UPDATE runners SET revoked = 1 WHERE id = ? AND owner_kind = ? AND owner_id = ?`, id, ownerKind, ownerID)
155
if err != nil {
156
return err
157
}
158
if n, _ := res.RowsAffected(); n == 0 {
159
return errNotFound
160
}
161
return nil
162
}
163
164
// UserRunnerPref is own_first of builtin_first.
165
func (s *Store) UserRunnerPref(ctx context.Context, userID int64) string {
166
var p string
167
if err := s.db.QueryRowContext(ctx, `SELECT runner_pref FROM users WHERE id = ?`, userID).Scan(&p); err != nil || p != RunnerPrefOwnFirst {
168
return RunnerPrefBuiltinFirst
169
}
170
return p
171
}
172
173
// SetUserRunnerPref bewaart de voorkeur.
174
func (s *Store) SetUserRunnerPref(ctx context.Context, userID int64, pref string) error {
175
if pref != RunnerPrefOwnFirst {
176
pref = RunnerPrefBuiltinFirst
177
}
178
_, err := s.db.ExecContext(ctx, `UPDATE users SET runner_pref = ? WHERE id = ?`, pref, userID)
179
return err
180
}
181
182
// GroupBuiltinFirst: "Probeer BlueGit runner eerst".
183
func (s *Store) GroupBuiltinFirst(ctx context.Context, groupID int64) bool {
184
var v int
185
_ = s.db.QueryRowContext(ctx, `SELECT builtin_first FROM groups WHERE id = ?`, groupID).Scan(&v)
186
return v == 1
187
}
188
189
// SetGroupBuiltinFirst zet de groepsinstelling.
190
func (s *Store) SetGroupBuiltinFirst(ctx context.Context, groupID int64, on bool) error {
191
v := 0
192
if on {
193
v = 1
194
}
195
_, err := s.db.ExecContext(ctx, `UPDATE groups SET builtin_first = ? WHERE id = ?`, v, groupID)
196
return err
197
}
198
199
// SetRunnerOptIn: maintainer userID staat zijn account-runners toe voor de groep.
200
func (s *Store) SetRunnerOptIn(ctx context.Context, groupID, userID int64, on bool) error {
201
if on {
202
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO group_runner_optin (group_id, user_id) VALUES (?, ?)`, groupID, userID)
203
return err
204
}
205
_, err := s.db.ExecContext(ctx, `DELETE FROM group_runner_optin WHERE group_id = ? AND user_id = ?`, groupID, userID)
206
return err
207
}
208
209
// RunnerOptIns zijn de gebruikers die hun runners voor de groep toestaan.
210
func (s *Store) RunnerOptIns(ctx context.Context, groupID int64) ([]int64, error) {
211
rows, err := s.db.QueryContext(ctx, `SELECT user_id FROM group_runner_optin WHERE group_id = ? ORDER BY user_id`, groupID)
212
if err != nil {
213
return nil, err
214
}
215
defer rows.Close()
216
var out []int64
217
for rows.Next() {
218
var id int64
219
if err := rows.Scan(&id); err != nil {
220
return nil, err
221
}
222
out = append(out, id)
223
}
224
return out, rows.Err()
225
}
226
227
// JobRun is de runner-administratie van één job.
228
type JobRun struct {
229
RunnerID int64
230
RunnerName string
231
OfferTier int
232
OfferAt time.Time
233
}
234
235
// JobRun leest de administratie; een lege JobRun als er niets is.
236
func (s *Store) JobRun(ctx context.Context, jobID int64) JobRun {
237
var jr JobRun
238
var at string
239
if err := s.db.QueryRowContext(ctx, `SELECT runner_id, runner_name, offer_tier, offer_at FROM job_runs WHERE job_id = ?`, jobID).Scan(&jr.RunnerID, &jr.RunnerName, &jr.OfferTier, &at); err != nil {
240
return JobRun{}
241
}
242
jr.OfferAt = parseTime(at)
243
return jr
244
}
245
246
// SetJobOffer zet de huidige trede in de runnervolgorde.
247
func (s *Store) SetJobOffer(ctx context.Context, jobID int64, tier int, at time.Time) error {
248
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, offer_tier, offer_at) VALUES (?, ?, ?)
249
ON CONFLICT(job_id) DO UPDATE SET offer_tier = excluded.offer_tier, offer_at = excluded.offer_at`, jobID, tier, at.UTC().Format(time.RFC3339Nano))
250
return err
251
}
252
253
// ClearJobOffer haalt het aanbod weg (bijvoorbeeld bij een retry).
254
func (s *Store) ClearJobOffer(ctx context.Context, jobID int64) error {
255
_, err := s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '', offer_tier = 0 WHERE job_id = ?`, jobID)
256
return err
257
}
258
259
// SetJobRunner noteert wie de job draait.
260
func (s *Store) SetJobRunner(ctx context.Context, jobID, runnerID int64, name string) error {
261
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, runner_id, runner_name) VALUES (?, ?, ?)
262
ON CONFLICT(job_id) DO UPDATE SET runner_id = excluded.runner_id, runner_name = excluded.runner_name`, jobID, runnerID, name)
263
return err
264
}
265
266
// OfferedJobs zijn wachtende jobs met een actief aanbod.
267
func (s *Store) OfferedJobs(ctx context.Context) ([]Job, map[int64]JobRun, error) {
268
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'queued' AND r.offer_at != '' ORDER BY j.id`)
269
if err != nil {
270
return nil, nil, err
271
}
272
var out []Job
273
for rows.Next() {
274
j, err := scanJob(rows)
275
if err != nil {
276
rows.Close()
277
return nil, nil, err
278
}
279
out = append(out, j)
280
}
281
rows.Close()
282
runs := map[int64]JobRun{}
283
for _, j := range out {
284
runs[j.ID] = s.JobRun(ctx, j.ID)
285
}
286
return out, runs, nil
287
}
288
289
// ClaimJob zet een wachtende job op running voor een runner. false als
290
// iemand anders hem al had.
291
func (s *Store) ClaimJob(ctx context.Context, jobID, runnerID int64, name string) (bool, error) {
292
ok, err := s.MarkJobRunning(ctx, jobID)
293
if err != nil || !ok {
294
return false, err
295
}
296
if err := s.SetJobRunner(ctx, jobID, runnerID, name); err != nil {
297
return true, err
298
}
299
_, err = s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '' WHERE job_id = ?`, jobID)
300
return true, err
301
}
302
303
// RemoteJob is een lopende job op een aparte runner.
304
type RemoteJob struct {
305
Job Job
306
RunnerID int64
307
}
308
309
// RemoteRunningJobs geeft lopende jobs die een aparte runner heeft.
310
func (s *Store) RemoteRunningJobs(ctx context.Context) ([]RemoteJob, error) {
311
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'running' AND r.runner_id > 0 ORDER BY j.id`)
312
if err != nil {
313
return nil, err
314
}
315
var out []RemoteJob
316
for rows.Next() {
317
j, err := scanJob(rows)
318
if err != nil {
319
rows.Close()
320
return nil, err
321
}
322
out = append(out, RemoteJob{Job: j})
323
}
324
rows.Close()
325
for i := range out {
326
out[i].RunnerID = s.JobRun(ctx, out[i].Job.ID).RunnerID
327
}
328
return out, nil
329
}
330
331
// JobForRunner geeft een job als die bij runnerID hoort (en loopt of net klaar is).
332
func (s *Store) JobForRunner(ctx context.Context, runnerID, jobID int64) (Job, error) {
333
if s.JobRun(ctx, jobID).RunnerID != runnerID || runnerID == 0 {
334
return Job{}, errNotFound
335
}
336
rows, err := s.db.QueryContext(ctx, jobSelect+` WHERE j.id = ?`, jobID)
337
if err != nil {
338
return Job{}, err
339
}
340
defer rows.Close()
341
if !rows.Next() {
342
return Job{}, errNotFound
343
}
344
return scanJob(rows)
345
}
346
347
// RunnerRunsRepo zegt of de runner nu een job van de repo draait.
348
func (s *Store) RunnerRunsRepo(ctx context.Context, runnerID, repoID int64) bool {
349
var n int
350
_ = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM jobs j JOIN job_runs r ON r.job_id = j.id WHERE r.runner_id = ? AND j.repo_id = ? AND j.status = 'running'`, runnerID, repoID).Scan(&n)
351
return n > 0 && runnerID > 0
352
}
@@ -0,0 +1,352 @@1
package store2
3
import (4
"context"5
"database/sql"6
"errors"7
"strings"8
"time"9
)10
11
// Runner is een aparte bluegit-runner op een eigen machine, gekoppeld aan12
// een account (owner_kind user) of een groep (owner_kind group).13
type Runner struct {14
ID int6415
OwnerKind string16
OwnerID int6417
Name string18
CreatedAt time.Time19
LastSeen time.Time20
Version string21
Revoked bool22
}23
24
// RunnerOnlineWindow: zo lang na de laatste heartbeat telt een runner als online.25
const RunnerOnlineWindow = 90 * time.Second26
27
// Online zegt of de runner recent contact had.28
func (r Runner) Online(now time.Time) bool {29
return !r.Revoked && !r.LastSeen.IsZero() && now.Sub(r.LastSeen) < RunnerOnlineWindow30
}31
32
// Runner-voorkeuren van een account.33
const (34
RunnerPrefOwnFirst = "own_first"35
RunnerPrefBuiltinFirst = "builtin_first"36
)37
38
// ErrRegToken is een onbekend, gebruikt of verlopen registratietoken.39
var ErrRegToken = errors.New("registration token invalid or expired")40
41
// CreateRunnerRegToken maakt een eenmalig registratietoken (24 uur geldig).42
func (s *Store) CreateRunnerRegToken(ctx context.Context, ownerKind string, ownerID, by int64) (string, error) {43
plain, err := newToken("bgrr_")44
if err != nil {45
return "", err46
}47
exp := time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)48
_, err = s.db.ExecContext(ctx, `INSERT INTO runner_reg_tokens (owner_kind, owner_id, token_hash, created_by, expires_at) VALUES (?, ?, ?, ?, ?)`,49
ownerKind, ownerID, hashToken(plain), by, exp)50
return plain, err51
}52
53
// RegisterRunner wisselt een registratietoken in voor een runner en zijn54
// eigen runner-token (bgr_). Alleen de hash wordt bewaard.55
func (s *Store) RegisterRunner(ctx context.Context, regToken, name, version string) (Runner, string, error) {56
tx, err := s.db.BeginTx(ctx, nil)57
if err != nil {58
return Runner{}, "", err59
}60
defer func() { _ = tx.Rollback() }()61
var id, ownerID, by int6462
var kind, exp string63
err = tx.QueryRowContext(ctx, `SELECT id, owner_kind, owner_id, created_by, expires_at FROM runner_reg_tokens WHERE token_hash = ? AND used = 0`, hashToken(regToken)).Scan(&id, &kind, &ownerID, &by, &exp)64
if err != nil || parseTime(exp).Before(time.Now()) {65
return Runner{}, "", ErrRegToken66
}67
if _, err := tx.ExecContext(ctx, `UPDATE runner_reg_tokens SET used = 1 WHERE id = ?`, id); err != nil {68
return Runner{}, "", err69
}70
plain, err := newToken("bgr_")71
if err != nil {72
return Runner{}, "", err73
}74
name = strings.TrimSpace(name)75
if name == "" {76
name = "runner"77
}78
if len(name) > 80 {79
name = name[:80]80
}81
at := now()82
res, err := tx.ExecContext(ctx, `INSERT INTO runners (owner_kind, owner_id, name, token_hash, created_by, created_at, last_seen, version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,83
kind, ownerID, name, hashToken(plain), by, at, at, version)84
if err != nil {85
return Runner{}, "", err86
}87
rid, _ := res.LastInsertId()88
if err := tx.Commit(); err != nil {89
return Runner{}, "", err90
}91
return Runner{ID: rid, OwnerKind: kind, OwnerID: ownerID, Name: name, CreatedAt: parseTime(at), LastSeen: parseTime(at), Version: version}, plain, nil92
}93
94
const runnerCols = `id, owner_kind, owner_id, name, created_at, last_seen, version, revoked`95
96
func scanRunner(sc interface{ Scan(...any) error }) (Runner, error) {97
var r Runner98
var created, seen string99
var revoked int100
if err := sc.Scan(&r.ID, &r.OwnerKind, &r.OwnerID, &r.Name, &created, &seen, &r.Version, &revoked); err != nil {101
return Runner{}, err102
}103
r.CreatedAt, r.LastSeen, r.Revoked = parseTime(created), parseTime(seen), revoked == 1104
return r, nil105
}106
107
// RunnerByToken geeft de runner bij een bgr_-token. Ingetrokken telt niet.108
func (s *Store) RunnerByToken(ctx context.Context, token string) (Runner, error) {109
if !strings.HasPrefix(token, "bgr_") {110
return Runner{}, errNotFound111
}112
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE token_hash = ? AND revoked = 0`, hashToken(token)))113
if errors.Is(err, sql.ErrNoRows) {114
return Runner{}, errNotFound115
}116
return r, err117
}118
119
// RunnerByID geeft één runner.120
func (s *Store) RunnerByID(ctx context.Context, id int64) (Runner, error) {121
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE id = ?`, id))122
if errors.Is(err, sql.ErrNoRows) {123
return Runner{}, errNotFound124
}125
return r, err126
}127
128
// TouchRunner is de heartbeat.129
func (s *Store) TouchRunner(ctx context.Context, id int64, version string) error {130
_, err := s.db.ExecContext(ctx, `UPDATE runners SET last_seen = ?, version = CASE WHEN ? = '' THEN version ELSE ? END WHERE id = ?`, now(), version, version, id)131
return err132
}133
134
// ListRunners geeft de niet-ingetrokken runners van een eigenaar.135
func (s *Store) ListRunners(ctx context.Context, ownerKind string, ownerID int64) ([]Runner, error) {136
rows, err := s.db.QueryContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE owner_kind = ? AND owner_id = ? AND revoked = 0 ORDER BY id`, ownerKind, ownerID)137
if err != nil {138
return nil, err139
}140
defer rows.Close()141
var out []Runner142
for rows.Next() {143
r, err := scanRunner(rows)144
if err != nil {145
return nil, err146
}147
out = append(out, r)148
}149
return out, rows.Err()150
}151
152
// RevokeRunner trekt een runner in; zijn token werkt daarna niet meer.153
func (s *Store) RevokeRunner(ctx context.Context, ownerKind string, ownerID, id int64) error {154
res, err := s.db.ExecContext(ctx, `UPDATE runners SET revoked = 1 WHERE id = ? AND owner_kind = ? AND owner_id = ?`, id, ownerKind, ownerID)155
if err != nil {156
return err157
}158
if n, _ := res.RowsAffected(); n == 0 {159
return errNotFound160
}161
return nil162
}163
164
// UserRunnerPref is own_first of builtin_first.165
func (s *Store) UserRunnerPref(ctx context.Context, userID int64) string {166
var p string167
if err := s.db.QueryRowContext(ctx, `SELECT runner_pref FROM users WHERE id = ?`, userID).Scan(&p); err != nil || p != RunnerPrefOwnFirst {168
return RunnerPrefBuiltinFirst169
}170
return p171
}172
173
// SetUserRunnerPref bewaart de voorkeur.174
func (s *Store) SetUserRunnerPref(ctx context.Context, userID int64, pref string) error {175
if pref != RunnerPrefOwnFirst {176
pref = RunnerPrefBuiltinFirst177
}178
_, err := s.db.ExecContext(ctx, `UPDATE users SET runner_pref = ? WHERE id = ?`, pref, userID)179
return err180
}181
182
// GroupBuiltinFirst: "Probeer BlueGit runner eerst".183
func (s *Store) GroupBuiltinFirst(ctx context.Context, groupID int64) bool {184
var v int185
_ = s.db.QueryRowContext(ctx, `SELECT builtin_first FROM groups WHERE id = ?`, groupID).Scan(&v)186
return v == 1187
}188
189
// SetGroupBuiltinFirst zet de groepsinstelling.190
func (s *Store) SetGroupBuiltinFirst(ctx context.Context, groupID int64, on bool) error {191
v := 0192
if on {193
v = 1194
}195
_, err := s.db.ExecContext(ctx, `UPDATE groups SET builtin_first = ? WHERE id = ?`, v, groupID)196
return err197
}198
199
// SetRunnerOptIn: maintainer userID staat zijn account-runners toe voor de groep.200
func (s *Store) SetRunnerOptIn(ctx context.Context, groupID, userID int64, on bool) error {201
if on {202
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO group_runner_optin (group_id, user_id) VALUES (?, ?)`, groupID, userID)203
return err204
}205
_, err := s.db.ExecContext(ctx, `DELETE FROM group_runner_optin WHERE group_id = ? AND user_id = ?`, groupID, userID)206
return err207
}208
209
// RunnerOptIns zijn de gebruikers die hun runners voor de groep toestaan.210
func (s *Store) RunnerOptIns(ctx context.Context, groupID int64) ([]int64, error) {211
rows, err := s.db.QueryContext(ctx, `SELECT user_id FROM group_runner_optin WHERE group_id = ? ORDER BY user_id`, groupID)212
if err != nil {213
return nil, err214
}215
defer rows.Close()216
var out []int64217
for rows.Next() {218
var id int64219
if err := rows.Scan(&id); err != nil {220
return nil, err221
}222
out = append(out, id)223
}224
return out, rows.Err()225
}226
227
// JobRun is de runner-administratie van één job.228
type JobRun struct {229
RunnerID int64230
RunnerName string231
OfferTier int232
OfferAt time.Time233
}234
235
// JobRun leest de administratie; een lege JobRun als er niets is.236
func (s *Store) JobRun(ctx context.Context, jobID int64) JobRun {237
var jr JobRun238
var at string239
if err := s.db.QueryRowContext(ctx, `SELECT runner_id, runner_name, offer_tier, offer_at FROM job_runs WHERE job_id = ?`, jobID).Scan(&jr.RunnerID, &jr.RunnerName, &jr.OfferTier, &at); err != nil {240
return JobRun{}241
}242
jr.OfferAt = parseTime(at)243
return jr244
}245
246
// SetJobOffer zet de huidige trede in de runnervolgorde.247
func (s *Store) SetJobOffer(ctx context.Context, jobID int64, tier int, at time.Time) error {248
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, offer_tier, offer_at) VALUES (?, ?, ?)249
ON CONFLICT(job_id) DO UPDATE SET offer_tier = excluded.offer_tier, offer_at = excluded.offer_at`, jobID, tier, at.UTC().Format(time.RFC3339Nano))250
return err251
}252
253
// ClearJobOffer haalt het aanbod weg (bijvoorbeeld bij een retry).254
func (s *Store) ClearJobOffer(ctx context.Context, jobID int64) error {255
_, err := s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '', offer_tier = 0 WHERE job_id = ?`, jobID)256
return err257
}258
259
// SetJobRunner noteert wie de job draait.260
func (s *Store) SetJobRunner(ctx context.Context, jobID, runnerID int64, name string) error {261
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, runner_id, runner_name) VALUES (?, ?, ?)262
ON CONFLICT(job_id) DO UPDATE SET runner_id = excluded.runner_id, runner_name = excluded.runner_name`, jobID, runnerID, name)263
return err264
}265
266
// OfferedJobs zijn wachtende jobs met een actief aanbod.267
func (s *Store) OfferedJobs(ctx context.Context) ([]Job, map[int64]JobRun, error) {268
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'queued' AND r.offer_at != '' ORDER BY j.id`)269
if err != nil {270
return nil, nil, err271
}272
var out []Job273
for rows.Next() {274
j, err := scanJob(rows)275
if err != nil {276
rows.Close()277
return nil, nil, err278
}279
out = append(out, j)280
}281
rows.Close()282
runs := map[int64]JobRun{}283
for _, j := range out {284
runs[j.ID] = s.JobRun(ctx, j.ID)285
}286
return out, runs, nil287
}288
289
// ClaimJob zet een wachtende job op running voor een runner. false als290
// iemand anders hem al had.291
func (s *Store) ClaimJob(ctx context.Context, jobID, runnerID int64, name string) (bool, error) {292
ok, err := s.MarkJobRunning(ctx, jobID)293
if err != nil || !ok {294
return false, err295
}296
if err := s.SetJobRunner(ctx, jobID, runnerID, name); err != nil {297
return true, err298
}299
_, err = s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '' WHERE job_id = ?`, jobID)300
return true, err301
}302
303
// RemoteJob is een lopende job op een aparte runner.304
type RemoteJob struct {305
Job Job306
RunnerID int64307
}308
309
// RemoteRunningJobs geeft lopende jobs die een aparte runner heeft.310
func (s *Store) RemoteRunningJobs(ctx context.Context) ([]RemoteJob, error) {311
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'running' AND r.runner_id > 0 ORDER BY j.id`)312
if err != nil {313
return nil, err314
}315
var out []RemoteJob316
for rows.Next() {317
j, err := scanJob(rows)318
if err != nil {319
rows.Close()320
return nil, err321
}322
out = append(out, RemoteJob{Job: j})323
}324
rows.Close()325
for i := range out {326
out[i].RunnerID = s.JobRun(ctx, out[i].Job.ID).RunnerID327
}328
return out, nil329
}330
331
// JobForRunner geeft een job als die bij runnerID hoort (en loopt of net klaar is).332
func (s *Store) JobForRunner(ctx context.Context, runnerID, jobID int64) (Job, error) {333
if s.JobRun(ctx, jobID).RunnerID != runnerID || runnerID == 0 {334
return Job{}, errNotFound335
}336
rows, err := s.db.QueryContext(ctx, jobSelect+` WHERE j.id = ?`, jobID)337
if err != nil {338
return Job{}, err339
}340
defer rows.Close()341
if !rows.Next() {342
return Job{}, errNotFound343
}344
return scanJob(rows)345
}346
347
// RunnerRunsRepo zegt of de runner nu een job van de repo draait.348
func (s *Store) RunnerRunsRepo(ctx context.Context, runnerID, repoID int64) bool {349
var n int350
_ = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM jobs j JOIN job_runs r ON r.job_id = j.id WHERE r.runner_id = ? AND j.repo_id = ? AND j.status = 'running'`, runnerID, repoID).Scan(&n)351
return n > 0 && runnerID > 0352
}@@ -102,6 +102,8 @@ func (s *Store) migrate() error {
102
102
`ALTER TABLE pipelines ADD COLUMN target_sha TEXT NOT NULL DEFAULT ''`,
103
103
`ALTER TABLE users ADD COLUMN display_name TEXT NOT NULL DEFAULT ''`,
104
104
`ALTER TABLE users ADD COLUMN bio TEXT NOT NULL DEFAULT ''`,
105
`ALTER TABLE users ADD COLUMN runner_pref TEXT NOT NULL DEFAULT 'builtin_first'`,
106
`ALTER TABLE groups ADD COLUMN builtin_first INTEGER NOT NULL DEFAULT 0`,
105
107
}
106
108
for _, q := range alters {
107
109
if _, err := s.db.Exec(q); err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column") {
@@ -115,6 +117,39 @@ func (s *Store) migrate() error {
115
117
}
116
118
117
119
const schema = `
120
CREATE TABLE IF NOT EXISTS runners (
121
id INTEGER PRIMARY KEY,
122
owner_kind TEXT NOT NULL,
123
owner_id INTEGER NOT NULL,
124
name TEXT NOT NULL,
125
token_hash TEXT NOT NULL UNIQUE,
126
created_by INTEGER NOT NULL DEFAULT 0,
127
created_at TEXT NOT NULL,
128
last_seen TEXT NOT NULL DEFAULT '',
129
version TEXT NOT NULL DEFAULT '',
130
revoked INTEGER NOT NULL DEFAULT 0
131
);
132
CREATE TABLE IF NOT EXISTS runner_reg_tokens (
133
id INTEGER PRIMARY KEY,
134
owner_kind TEXT NOT NULL,
135
owner_id INTEGER NOT NULL,
136
token_hash TEXT NOT NULL UNIQUE,
137
created_by INTEGER NOT NULL,
138
expires_at TEXT NOT NULL,
139
used INTEGER NOT NULL DEFAULT 0
140
);
141
CREATE TABLE IF NOT EXISTS group_runner_optin (
142
group_id INTEGER NOT NULL,
143
user_id INTEGER NOT NULL,
144
PRIMARY KEY (group_id, user_id)
145
);
146
CREATE TABLE IF NOT EXISTS job_runs (
147
job_id INTEGER PRIMARY KEY,
148
runner_id INTEGER NOT NULL DEFAULT 0,
149
runner_name TEXT NOT NULL DEFAULT '',
150
offer_tier INTEGER NOT NULL DEFAULT 0,
151
offer_at TEXT NOT NULL DEFAULT ''
152
);
118
153
CREATE TABLE IF NOT EXISTS users (
119
154
id INTEGER PRIMARY KEY,
120
155
auth_id INTEGER NOT NULL UNIQUE,
@@ -102,6 +102,8 @@ func (s *Store) migrate() error {102
`ALTER TABLE pipelines ADD COLUMN target_sha TEXT NOT NULL DEFAULT ''`,102
`ALTER TABLE pipelines ADD COLUMN target_sha TEXT NOT NULL DEFAULT ''`,103
`ALTER TABLE users ADD COLUMN display_name TEXT NOT NULL DEFAULT ''`,103
`ALTER TABLE users ADD COLUMN display_name TEXT NOT NULL DEFAULT ''`,104
`ALTER TABLE users ADD COLUMN bio TEXT NOT NULL DEFAULT ''`,104
`ALTER TABLE users ADD COLUMN bio TEXT NOT NULL DEFAULT ''`,105
`ALTER TABLE users ADD COLUMN runner_pref TEXT NOT NULL DEFAULT 'builtin_first'`,106
`ALTER TABLE groups ADD COLUMN builtin_first INTEGER NOT NULL DEFAULT 0`,105
}107
}106
for _, q := range alters {108
for _, q := range alters {107
if _, err := s.db.Exec(q); err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column") {109
if _, err := s.db.Exec(q); err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column") {@@ -115,6 +117,39 @@ func (s *Store) migrate() error {115
}117
}116
118
117
const schema = `119
const schema = `120
CREATE TABLE IF NOT EXISTS runners (121
id INTEGER PRIMARY KEY,122
owner_kind TEXT NOT NULL,123
owner_id INTEGER NOT NULL,124
name TEXT NOT NULL,125
token_hash TEXT NOT NULL UNIQUE,126
created_by INTEGER NOT NULL DEFAULT 0,127
created_at TEXT NOT NULL,128
last_seen TEXT NOT NULL DEFAULT '',129
version TEXT NOT NULL DEFAULT '',130
revoked INTEGER NOT NULL DEFAULT 0131
);132
CREATE TABLE IF NOT EXISTS runner_reg_tokens (133
id INTEGER PRIMARY KEY,134
owner_kind TEXT NOT NULL,135
owner_id INTEGER NOT NULL,136
token_hash TEXT NOT NULL UNIQUE,137
created_by INTEGER NOT NULL,138
expires_at TEXT NOT NULL,139
used INTEGER NOT NULL DEFAULT 0140
);141
CREATE TABLE IF NOT EXISTS group_runner_optin (142
group_id INTEGER NOT NULL,143
user_id INTEGER NOT NULL,144
PRIMARY KEY (group_id, user_id)145
);146
CREATE TABLE IF NOT EXISTS job_runs (147
job_id INTEGER PRIMARY KEY,148
runner_id INTEGER NOT NULL DEFAULT 0,149
runner_name TEXT NOT NULL DEFAULT '',150
offer_tier INTEGER NOT NULL DEFAULT 0,151
offer_at TEXT NOT NULL DEFAULT ''152
);118
CREATE TABLE IF NOT EXISTS users (153
CREATE TABLE IF NOT EXISTS users (119
id INTEGER PRIMARY KEY,154
id INTEGER PRIMARY KEY,120
auth_id INTEGER NOT NULL UNIQUE,155
auth_id INTEGER NOT NULL UNIQUE,