Merged
Aparte runner: bluegit-runner (AUR), runnervolgorde per account/groep
timfa
wants to merge
5 commits
from
cursor/runner-module
into
stage
Overview Commits 5 Pipelines 1 Changes 29
Nieuwe module bluegit-runner (cmd/bluegit-runner) naast de ingebouwde runner.
- Registratie met eenmalig registratietoken uit de UI (Instellingen → Runners, Groep → Instellingen → Runners); runner krijgt eigen
bgr_token, alleen hash opgeslagen. Intrekken in de UI. - Uitgaande long-poll, heartbeat, online/offline; log streamen, artifacts in/uit; cache lokaal op de runner.
- Volgorde: account
builtin_first(standaard) ofown_first; groep: groepsrunners → opt-in maintainer-runners → BlueGit-runner, of BlueGit eerst met "Probeer BlueGit runner eerst". Doorschuiven na 20 s of bij offline. - Privacy: account-runner alleen repo's die de eigenaar mag lezen, groepsrunner alleen eigen groep; runner-token mag alleen clonen voor een job die hij draait. Job-token rechten ongewijzigd.
- Job-UI en job-JSON tonen de runner.
- Packaging: packaging/runner, packaging/aur-runner, CI-job publish_aur_runner (zelfde AUR-SSH-geheim).
- Tests: routing/privacy unit tests; e2e die de echte binary registreert en een pipeline met artifacts draait; ingebouwde runner e2e; fallback-test. Fake docker, draait in CI.
- Docs: docs/runner.md.
Activity
- timfa Pipeline #206 is Passed 24 October 2026
- timfa opened this merge request 24 October 2026
- timfa merged this merge request 24 October 2026
Commits
- runner module: docs link, gofmt
5a6e2582ddtimfa - runner module: packaging, AUR job, docs
f017c6065atimfa - runner module: routing unit tests and e2e tests (builtin, separate runner, fallback)
ef342a994ftimfa - wip runner module: API, UI, bluegit-runner binary
a7578bb106timfa - wip runner module: store, routing, remote core
49c37d340btimfa
Pipelines
- #206 Passed
Changes
@@ -81,6 +81,20 @@ jobs:
81
81
- ./scripts/aur-ssh-setup.sh
82
82
- ./scripts/publish-aur.sh
83
83
84
# Het aparte runner-pakket. Zelfde AUR-SSH-geheim als publish_aur.
85
publish_aur_runner:
86
stage: release
87
image: archlinux/archlinux:base-devel
88
extends: .on_master
89
needs: [build]
90
allow_failure: true
91
script:
92
- pacman -Syu --noconfirm
93
- pacman -S --noconfirm --needed git openssh curl coreutils shadow util-linux
94
- chmod +x scripts/aur-ssh-setup.sh scripts/publish-aur.sh
95
- ./scripts/aur-ssh-setup.sh
96
- AUR_PKG=bluegit-runner ./scripts/publish-aur.sh
97
84
98
# Laatste job. needs: [publish_aur] geldt ook als die job met
85
99
# allow_failure mislukt. Alleen de POST; de job eindigt meteen groen.
86
100
# De updater bouwt zelf uit git; dit artifact gaat niet mee.
@@ -90,7 +104,7 @@ jobs:
90
104
stage: release
91
105
image: bluegit/base
92
106
extends: .on_master
93
needs: [publish_aur]
107
needs: [publish_aur, publish_aur_runner]
94
108
script:
95
109
- chmod +x scripts/self-update.sh
96
110
- ./scripts/self-update.sh
@@ -81,6 +81,20 @@ jobs:81
- ./scripts/aur-ssh-setup.sh81
- ./scripts/aur-ssh-setup.sh82
- ./scripts/publish-aur.sh82
- ./scripts/publish-aur.sh83
83
84
# Het aparte runner-pakket. Zelfde AUR-SSH-geheim als publish_aur.85
publish_aur_runner:86
stage: release87
image: archlinux/archlinux:base-devel88
extends: .on_master89
needs: [build]90
allow_failure: true91
script:92
- pacman -Syu --noconfirm93
- pacman -S --noconfirm --needed git openssh curl coreutils shadow util-linux94
- chmod +x scripts/aur-ssh-setup.sh scripts/publish-aur.sh95
- ./scripts/aur-ssh-setup.sh96
- AUR_PKG=bluegit-runner ./scripts/publish-aur.sh97
84
# Laatste job. needs: [publish_aur] geldt ook als die job met98
# Laatste job. needs: [publish_aur] geldt ook als die job met85
# allow_failure mislukt. Alleen de POST; de job eindigt meteen groen.99
# allow_failure mislukt. Alleen de POST; de job eindigt meteen groen.86
# De updater bouwt zelf uit git; dit artifact gaat niet mee.100
# De updater bouwt zelf uit git; dit artifact gaat niet mee.@@ -90,7 +104,7 @@ jobs:90
stage: release104
stage: release91
image: bluegit/base105
image: bluegit/base92
extends: .on_master106
extends: .on_master93
needs: [publish_aur]107
needs: [publish_aur, publish_aur_runner]94
script:108
script:95
- chmod +x scripts/self-update.sh109
- chmod +x scripts/self-update.sh96
- ./scripts/self-update.sh110
- ./scripts/self-update.sh@@ -0,0 +1,447 @@
1
// Command bluegit-runner draait BlueGit CI-jobs op een eigen machine.
2
// Hij maakt alleen uitgaande HTTPS-verbindingen (long-poll), er hoeft
3
// geen poort open.
4
package main
5
6
import (
7
"bytes"
8
"context"
9
"encoding/json"
10
"errors"
11
"flag"
12
"fmt"
13
"io"
14
"log/slog"
15
"net/http"
16
"os"
17
"os/exec"
18
"os/signal"
19
"os/user"
20
"path/filepath"
21
"strconv"
22
"strings"
23
"sync"
24
"syscall"
25
"time"
26
27
"gopkg.in/yaml.v3"
28
29
"gitlab.com/Timfa/bluegit/internal/runner"
30
)
31
32
var version = "dev"
33
34
const defaultConfig = "/etc/bluegit-runner/config.yml"
35
36
// Config staat in /etc/bluegit-runner/config.yml.
37
type Config struct {
38
URL string `yaml:"url"`
39
Token string `yaml:"token"`
40
Name string `yaml:"name"`
41
Docker string `yaml:"docker"`
42
Git string `yaml:"git"`
43
WorkDir string `yaml:"work_dir"`
44
Network string `yaml:"network"`
45
Concurrency int `yaml:"concurrency"`
46
}
47
48
func (c *Config) defaults() {
49
if c.Docker == "" {
50
c.Docker = "docker"
51
}
52
if c.Git == "" {
53
c.Git = "git"
54
}
55
if c.WorkDir == "" {
56
c.WorkDir = "/var/lib/bluegit-runner"
57
}
58
if c.Network == "" {
59
c.Network = "bridge"
60
}
61
if c.Concurrency < 1 {
62
c.Concurrency = 1
63
}
64
c.URL = strings.TrimRight(c.URL, "/")
65
}
66
67
func main() {
68
if len(os.Args) < 2 {
69
usage()
70
os.Exit(2)
71
}
72
var err error
73
switch os.Args[1] {
74
case "register":
75
err = cmdRegister(os.Args[2:])
76
case "run":
77
err = cmdRun(os.Args[2:])
78
case "version", "--version":
79
fmt.Println(version)
80
default:
81
usage()
82
os.Exit(2)
83
}
84
if err != nil {
85
fmt.Fprintln(os.Stderr, "bluegit-runner:", err)
86
os.Exit(1)
87
}
88
}
89
90
func usage() {
91
fmt.Fprintln(os.Stderr, `usage:
92
bluegit-runner register --url https://git.example.com --token bgrr_... [--name NAME] [--config PATH]
93
bluegit-runner run [--config PATH]
94
bluegit-runner version`)
95
}
96
97
func loadConfig(path string) (Config, error) {
98
var c Config
99
b, err := os.ReadFile(path)
100
if err != nil {
101
return c, err
102
}
103
if err := yaml.Unmarshal(b, &c); err != nil {
104
return c, err
105
}
106
c.defaults()
107
if c.URL == "" || c.Token == "" {
108
return c, fmt.Errorf("%s: url and token are required; run bluegit-runner register first", path)
109
}
110
return c, nil
111
}
112
113
func cmdRegister(args []string) error {
114
fs := flag.NewFlagSet("register", flag.ExitOnError)
115
url := fs.String("url", "", "BlueGit base URL")
116
token := fs.String("token", "", "registration token from the BlueGit UI")
117
name := fs.String("name", "", "runner name (default: hostname)")
118
cfgPath := fs.String("config", defaultConfig, "config file")
119
_ = fs.Parse(args)
120
if *token == "" {
121
*token = os.Getenv("BLUEGIT_RUNNER_REGISTRATION_TOKEN")
122
}
123
if *url == "" || *token == "" {
124
return errors.New("--url and --token are required")
125
}
126
if *name == "" {
127
*name, _ = os.Hostname()
128
}
129
var existing Config
130
if b, err := os.ReadFile(*cfgPath); err == nil {
131
_ = yaml.Unmarshal(b, &existing)
132
}
133
body, _ := json.Marshal(map[string]string{"token": *token, "name": *name, "version": version})
134
resp, err := http.Post(strings.TrimRight(*url, "/")+"/api/v1/runners/register", "application/json", bytes.NewReader(body))
135
if err != nil {
136
return err
137
}
138
defer resp.Body.Close()
139
var out struct {
140
ID int64 `json:"id"`
141
Name string `json:"name"`
142
Token string `json:"token"`
143
Message string `json:"message"`
144
}
145
_ = json.NewDecoder(resp.Body).Decode(&out)
146
if resp.StatusCode != http.StatusCreated || out.Token == "" {
147
return fmt.Errorf("registration failed (%d): %s", resp.StatusCode, out.Message)
148
}
149
existing.URL, existing.Token, existing.Name = strings.TrimRight(*url, "/"), out.Token, out.Name
150
b, err := yaml.Marshal(existing)
151
if err != nil {
152
return err
153
}
154
if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil {
155
return err
156
}
157
if err := os.WriteFile(*cfgPath, b, 0o600); err != nil {
158
return err
159
}
160
if os.Geteuid() == 0 {
161
if u, err := user.Lookup("bluegit-runner"); err == nil {
162
uid, _ := strconv.Atoi(u.Uid)
163
gid, _ := strconv.Atoi(u.Gid)
164
_ = os.Chown(*cfgPath, uid, gid)
165
}
166
}
167
fmt.Printf("registered runner %q (id %d); config written to %s\n", out.Name, out.ID, *cfgPath)
168
// Met het AUR-pakket: de service is al enabled en wacht op de config.
169
if os.Geteuid() == 0 && *cfgPath == defaultConfig {
170
if _, err := os.Stat("/usr/lib/systemd/system/bluegit-runner.service"); err == nil {
171
if err := exec.Command("systemctl", "enable", "--now", "bluegit-runner.service").Run(); err == nil {
172
_ = exec.Command("systemctl", "restart", "bluegit-runner.service").Run()
173
fmt.Println("bluegit-runner.service started")
174
return nil
175
}
176
}
177
}
178
fmt.Println("start it with: sudo systemctl enable --now bluegit-runner")
179
return nil
180
}
181
182
type client struct {
183
cfg Config
184
http *http.Client
185
log *slog.Logger
186
}
187
188
func (c *client) do(ctx context.Context, method, path string, body io.Reader, ctype string) (*http.Response, error) {
189
req, err := http.NewRequestWithContext(ctx, method, c.cfg.URL+path, body)
190
if err != nil {
191
return nil, err
192
}
193
req.Header.Set("Authorization", "Bearer "+c.cfg.Token)
194
req.Header.Set("X-Runner-Version", version)
195
if ctype != "" {
196
req.Header.Set("Content-Type", ctype)
197
}
198
return c.http.Do(req)
199
}
200
201
func (c *client) postJSON(ctx context.Context, path string, in, out any) error {
202
b, _ := json.Marshal(in)
203
resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b), "application/json")
204
if err != nil {
205
return err
206
}
207
defer resp.Body.Close()
208
if resp.StatusCode >= 300 {
209
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
210
return fmt.Errorf("%s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(msg)))
211
}
212
if out != nil {
213
return json.NewDecoder(resp.Body).Decode(out)
214
}
215
return nil
216
}
217
218
func cmdRun(args []string) error {
219
fs := flag.NewFlagSet("run", flag.ExitOnError)
220
cfgPath := fs.String("config", defaultConfig, "config file")
221
once := fs.Bool("once", false, "run at most one job, then exit (for tests)")
222
_ = fs.Parse(args)
223
cfg, err := loadConfig(*cfgPath)
224
if err != nil {
225
return err
226
}
227
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
228
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
229
defer stop()
230
c := &client{cfg: cfg, http: &http.Client{Timeout: 0}, log: log}
231
if err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]string{}, nil); err != nil {
232
return fmt.Errorf("cannot reach BlueGit: %w", err)
233
}
234
log.Info("bluegit-runner started", "url", cfg.URL, "name", cfg.Name, "version", version, "concurrency", cfg.Concurrency)
235
go func() {
236
t := time.NewTicker(30 * time.Second)
237
defer t.Stop()
238
for {
239
select {
240
case <-ctx.Done():
241
return
242
case <-t.C:
243
if err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]string{}, nil); err != nil && ctx.Err() == nil {
244
log.Warn("heartbeat", "err", err)
245
}
246
}
247
}
248
}()
249
var wg sync.WaitGroup
250
for i := 0; i < cfg.Concurrency; i++ {
251
wg.Add(1)
252
go func() {
253
defer wg.Done()
254
for ctx.Err() == nil {
255
ran, err := c.poll(ctx)
256
if err != nil && ctx.Err() == nil {
257
log.Warn("poll", "err", err)
258
select {
259
case <-ctx.Done():
260
case <-time.After(5 * time.Second):
261
}
262
}
263
if ran && *once {
264
stop()
265
}
266
}
267
}()
268
}
269
wg.Wait()
270
return nil
271
}
272
273
func (c *client) poll(ctx context.Context) (bool, error) {
274
pctx, cancel := context.WithTimeout(ctx, 60*time.Second)
275
defer cancel()
276
resp, err := c.do(pctx, http.MethodPost, "/api/v1/runner/jobs/request", strings.NewReader("{}"), "application/json")
277
if err != nil {
278
return false, err
279
}
280
defer resp.Body.Close()
281
if resp.StatusCode == http.StatusNoContent {
282
return false, nil
283
}
284
if resp.StatusCode != http.StatusOK {
285
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
286
return false, fmt.Errorf("request: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
287
}
288
var pl runner.RemotePayload
289
if err := json.NewDecoder(resp.Body).Decode(&pl); err != nil {
290
return false, err
291
}
292
c.runJob(ctx, pl)
293
return true, nil
294
}
295
296
// logBuffer verzamelt logregels en stuurt ze elke seconde door.
297
type logBuffer struct {
298
mu sync.Mutex
299
buf strings.Builder
300
}
301
302
func (l *logBuffer) add(s string) {
303
l.mu.Lock()
304
l.buf.WriteString(s)
305
l.mu.Unlock()
306
}
307
308
func (l *logBuffer) take() string {
309
l.mu.Lock()
310
defer l.mu.Unlock()
311
s := l.buf.String()
312
l.buf.Reset()
313
return s
314
}
315
316
func (c *client) runJob(parent context.Context, pl runner.RemotePayload) {
317
c.log.Info("job", "id", pl.JobID, "name", pl.Job.Name)
318
base := fmt.Sprintf("/api/v1/runner/jobs/%d", pl.JobID)
319
ctx, cancel := context.WithCancel(parent)
320
defer cancel()
321
runCtx := ctx
322
if pl.Timeout > 0 {
323
var tc context.CancelFunc
324
runCtx, tc = context.WithTimeout(ctx, time.Duration(pl.Timeout)*time.Second)
325
defer tc()
326
}
327
var canceled bool
328
var cmu sync.Mutex
329
logs := &logBuffer{}
330
send := func() {
331
chunk := logs.take()
332
var out struct {
333
Cancel bool `json:"cancel"`
334
}
335
sctx, sc := context.WithTimeout(context.Background(), 30*time.Second)
336
defer sc()
337
err := func() error {
338
resp, err := c.do(sctx, http.MethodPost, base+"/log", strings.NewReader(chunk), "text/plain")
339
if err != nil {
340
return err
341
}
342
defer resp.Body.Close()
343
if resp.StatusCode != http.StatusOK {
344
return fmt.Errorf("log: %d", resp.StatusCode)
345
}
346
return json.NewDecoder(resp.Body).Decode(&out)
347
}()
348
if err != nil {
349
// Bij een fout gaat de chunk terug in de buffer voor de volgende poging.
350
if chunk != "" {
351
logs.mu.Lock()
352
rest := logs.buf.String()
353
logs.buf.Reset()
354
logs.buf.WriteString(chunk + rest)
355
logs.mu.Unlock()
356
}
357
return
358
}
359
if out.Cancel {
360
cmu.Lock()
361
canceled = true
362
cmu.Unlock()
363
cancel()
364
}
365
}
366
done := make(chan struct{})
367
flushed := make(chan struct{})
368
go func() {
369
defer close(flushed)
370
t := time.NewTicker(time.Second)
371
defer t.Stop()
372
for {
373
select {
374
case <-done:
375
return
376
case <-t.C:
377
send()
378
}
379
}
380
}()
381
r := &runner.Runner{GitBin: c.cfg.Git, Docker: c.cfg.Docker, Storage: c.cfg.WorkDir, Network: c.cfg.Network, Name: c.cfg.Name}
382
work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))
383
defer os.RemoveAll(work)
384
clone := func(ctx context.Context, dir string) error {
385
cmd := exec.CommandContext(ctx, c.cfg.Git, "clone", "--quiet", pl.CloneURL, dir)
386
// Het runner-token via de omgeving, niet in de proceslijst.
387
cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_COUNT=1",
388
"GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=Authorization: Bearer "+c.cfg.Token)
389
out, err := cmd.CombinedOutput()
390
if err != nil {
391
return fmt.Errorf("%s", strings.ReplaceAll(strings.TrimSpace(string(out)), c.cfg.Token, "[MASKED]"))
392
}
393
return nil
394
}
395
inputs := func(ctx context.Context, dir string) error {
396
resp, err := c.do(ctx, http.MethodGet, base+"/inputs", nil, "")
397
if err != nil {
398
return err
399
}
400
defer resp.Body.Close()
401
if resp.StatusCode != http.StatusOK {
402
return fmt.Errorf("inputs: %d", resp.StatusCode)
403
}
404
return runner.ReadTar(resp.Body, dir, nil, 0)
405
}
406
err := r.RunRemote(runCtx, pl, work, logs.add, clone, inputs)
407
if err == nil && len(pl.Job.ArtifactPaths) > 0 {
408
err = c.upload(runCtx, base, work, pl.Job.ArtifactPaths)
409
}
410
close(done)
411
<-flushed
412
send()
413
status, msg := "success", ""
414
cmu.Lock()
415
wasCanceled := canceled
416
cmu.Unlock()
417
switch {
418
case err == nil:
419
case wasCanceled:
420
status = "canceled"
421
case errors.Is(runCtx.Err(), context.DeadlineExceeded):
422
status, msg = "failed", "job_execution_timeout: "+err.Error()
423
default:
424
status, msg = "failed", err.Error()
425
}
426
fctx, fc := context.WithTimeout(context.Background(), time.Minute)
427
defer fc()
428
if ferr := c.postJSON(fctx, base+"/finish", map[string]string{"status": status, "message": msg}, nil); ferr != nil {
429
c.log.Error("finish", "job", pl.JobID, "err", ferr)
430
}
431
c.log.Info("job done", "id", pl.JobID, "status", status)
432
}
433
434
func (c *client) upload(ctx context.Context, base, work string, paths []string) error {
435
pr, pw := io.Pipe()
436
go func() { pw.CloseWithError(runner.WriteTar(pw, work, paths)) }()
437
resp, err := c.do(ctx, http.MethodPut, base+"/artifacts", pr, "application/gzip")
438
if err != nil {
439
return fmt.Errorf("artifacts upload: %w", err)
440
}
441
defer resp.Body.Close()
442
if resp.StatusCode != http.StatusOK {
443
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
444
return fmt.Errorf("artifacts upload: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))
445
}
446
return nil
447
}
@@ -0,0 +1,447 @@1
// Command bluegit-runner draait BlueGit CI-jobs op een eigen machine.2
// Hij maakt alleen uitgaande HTTPS-verbindingen (long-poll), er hoeft3
// geen poort open.4
package main5
6
import (7
"bytes"8
"context"9
"encoding/json"10
"errors"11
"flag"12
"fmt"13
"io"14
"log/slog"15
"net/http"16
"os"17
"os/exec"18
"os/signal"19
"os/user"20
"path/filepath"21
"strconv"22
"strings"23
"sync"24
"syscall"25
"time"26
27
"gopkg.in/yaml.v3"28
29
"gitlab.com/Timfa/bluegit/internal/runner"30
)31
32
var version = "dev"33
34
const defaultConfig = "/etc/bluegit-runner/config.yml"35
36
// Config staat in /etc/bluegit-runner/config.yml.37
type Config struct {38
URL string `yaml:"url"`39
Token string `yaml:"token"`40
Name string `yaml:"name"`41
Docker string `yaml:"docker"`42
Git string `yaml:"git"`43
WorkDir string `yaml:"work_dir"`44
Network string `yaml:"network"`45
Concurrency int `yaml:"concurrency"`46
}47
48
func (c *Config) defaults() {49
if c.Docker == "" {50
c.Docker = "docker"51
}52
if c.Git == "" {53
c.Git = "git"54
}55
if c.WorkDir == "" {56
c.WorkDir = "/var/lib/bluegit-runner"57
}58
if c.Network == "" {59
c.Network = "bridge"60
}61
if c.Concurrency < 1 {62
c.Concurrency = 163
}64
c.URL = strings.TrimRight(c.URL, "/")65
}66
67
func main() {68
if len(os.Args) < 2 {69
usage()70
os.Exit(2)71
}72
var err error73
switch os.Args[1] {74
case "register":75
err = cmdRegister(os.Args[2:])76
case "run":77
err = cmdRun(os.Args[2:])78
case "version", "--version":79
fmt.Println(version)80
default:81
usage()82
os.Exit(2)83
}84
if err != nil {85
fmt.Fprintln(os.Stderr, "bluegit-runner:", err)86
os.Exit(1)87
}88
}89
90
func usage() {91
fmt.Fprintln(os.Stderr, `usage:92
bluegit-runner register --url https://git.example.com --token bgrr_... [--name NAME] [--config PATH]93
bluegit-runner run [--config PATH]94
bluegit-runner version`)95
}96
97
func loadConfig(path string) (Config, error) {98
var c Config99
b, err := os.ReadFile(path)100
if err != nil {101
return c, err102
}103
if err := yaml.Unmarshal(b, &c); err != nil {104
return c, err105
}106
c.defaults()107
if c.URL == "" || c.Token == "" {108
return c, fmt.Errorf("%s: url and token are required; run bluegit-runner register first", path)109
}110
return c, nil111
}112
113
func cmdRegister(args []string) error {114
fs := flag.NewFlagSet("register", flag.ExitOnError)115
url := fs.String("url", "", "BlueGit base URL")116
token := fs.String("token", "", "registration token from the BlueGit UI")117
name := fs.String("name", "", "runner name (default: hostname)")118
cfgPath := fs.String("config", defaultConfig, "config file")119
_ = fs.Parse(args)120
if *token == "" {121
*token = os.Getenv("BLUEGIT_RUNNER_REGISTRATION_TOKEN")122
}123
if *url == "" || *token == "" {124
return errors.New("--url and --token are required")125
}126
if *name == "" {127
*name, _ = os.Hostname()128
}129
var existing Config130
if b, err := os.ReadFile(*cfgPath); err == nil {131
_ = yaml.Unmarshal(b, &existing)132
}133
body, _ := json.Marshal(map[string]string{"token": *token, "name": *name, "version": version})134
resp, err := http.Post(strings.TrimRight(*url, "/")+"/api/v1/runners/register", "application/json", bytes.NewReader(body))135
if err != nil {136
return err137
}138
defer resp.Body.Close()139
var out struct {140
ID int64 `json:"id"`141
Name string `json:"name"`142
Token string `json:"token"`143
Message string `json:"message"`144
}145
_ = json.NewDecoder(resp.Body).Decode(&out)146
if resp.StatusCode != http.StatusCreated || out.Token == "" {147
return fmt.Errorf("registration failed (%d): %s", resp.StatusCode, out.Message)148
}149
existing.URL, existing.Token, existing.Name = strings.TrimRight(*url, "/"), out.Token, out.Name150
b, err := yaml.Marshal(existing)151
if err != nil {152
return err153
}154
if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil {155
return err156
}157
if err := os.WriteFile(*cfgPath, b, 0o600); err != nil {158
return err159
}160
if os.Geteuid() == 0 {161
if u, err := user.Lookup("bluegit-runner"); err == nil {162
uid, _ := strconv.Atoi(u.Uid)163
gid, _ := strconv.Atoi(u.Gid)164
_ = os.Chown(*cfgPath, uid, gid)165
}166
}167
fmt.Printf("registered runner %q (id %d); config written to %s\n", out.Name, out.ID, *cfgPath)168
// Met het AUR-pakket: de service is al enabled en wacht op de config.169
if os.Geteuid() == 0 && *cfgPath == defaultConfig {170
if _, err := os.Stat("/usr/lib/systemd/system/bluegit-runner.service"); err == nil {171
if err := exec.Command("systemctl", "enable", "--now", "bluegit-runner.service").Run(); err == nil {172
_ = exec.Command("systemctl", "restart", "bluegit-runner.service").Run()173
fmt.Println("bluegit-runner.service started")174
return nil175
}176
}177
}178
fmt.Println("start it with: sudo systemctl enable --now bluegit-runner")179
return nil180
}181
182
type client struct {183
cfg Config184
http *http.Client185
log *slog.Logger186
}187
188
func (c *client) do(ctx context.Context, method, path string, body io.Reader, ctype string) (*http.Response, error) {189
req, err := http.NewRequestWithContext(ctx, method, c.cfg.URL+path, body)190
if err != nil {191
return nil, err192
}193
req.Header.Set("Authorization", "Bearer "+c.cfg.Token)194
req.Header.Set("X-Runner-Version", version)195
if ctype != "" {196
req.Header.Set("Content-Type", ctype)197
}198
return c.http.Do(req)199
}200
201
func (c *client) postJSON(ctx context.Context, path string, in, out any) error {202
b, _ := json.Marshal(in)203
resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b), "application/json")204
if err != nil {205
return err206
}207
defer resp.Body.Close()208
if resp.StatusCode >= 300 {209
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))210
return fmt.Errorf("%s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(msg)))211
}212
if out != nil {213
return json.NewDecoder(resp.Body).Decode(out)214
}215
return nil216
}217
218
func cmdRun(args []string) error {219
fs := flag.NewFlagSet("run", flag.ExitOnError)220
cfgPath := fs.String("config", defaultConfig, "config file")221
once := fs.Bool("once", false, "run at most one job, then exit (for tests)")222
_ = fs.Parse(args)223
cfg, err := loadConfig(*cfgPath)224
if err != nil {225
return err226
}227
log := slog.New(slog.NewTextHandler(os.Stderr, nil))228
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)229
defer stop()230
c := &client{cfg: cfg, http: &http.Client{Timeout: 0}, log: log}231
if err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]string{}, nil); err != nil {232
return fmt.Errorf("cannot reach BlueGit: %w", err)233
}234
log.Info("bluegit-runner started", "url", cfg.URL, "name", cfg.Name, "version", version, "concurrency", cfg.Concurrency)235
go func() {236
t := time.NewTicker(30 * time.Second)237
defer t.Stop()238
for {239
select {240
case <-ctx.Done():241
return242
case <-t.C:243
if err := c.postJSON(ctx, "/api/v1/runner/heartbeat", map[string]string{}, nil); err != nil && ctx.Err() == nil {244
log.Warn("heartbeat", "err", err)245
}246
}247
}248
}()249
var wg sync.WaitGroup250
for i := 0; i < cfg.Concurrency; i++ {251
wg.Add(1)252
go func() {253
defer wg.Done()254
for ctx.Err() == nil {255
ran, err := c.poll(ctx)256
if err != nil && ctx.Err() == nil {257
log.Warn("poll", "err", err)258
select {259
case <-ctx.Done():260
case <-time.After(5 * time.Second):261
}262
}263
if ran && *once {264
stop()265
}266
}267
}()268
}269
wg.Wait()270
return nil271
}272
273
func (c *client) poll(ctx context.Context) (bool, error) {274
pctx, cancel := context.WithTimeout(ctx, 60*time.Second)275
defer cancel()276
resp, err := c.do(pctx, http.MethodPost, "/api/v1/runner/jobs/request", strings.NewReader("{}"), "application/json")277
if err != nil {278
return false, err279
}280
defer resp.Body.Close()281
if resp.StatusCode == http.StatusNoContent {282
return false, nil283
}284
if resp.StatusCode != http.StatusOK {285
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))286
return false, fmt.Errorf("request: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))287
}288
var pl runner.RemotePayload289
if err := json.NewDecoder(resp.Body).Decode(&pl); err != nil {290
return false, err291
}292
c.runJob(ctx, pl)293
return true, nil294
}295
296
// logBuffer verzamelt logregels en stuurt ze elke seconde door.297
type logBuffer struct {298
mu sync.Mutex299
buf strings.Builder300
}301
302
func (l *logBuffer) add(s string) {303
l.mu.Lock()304
l.buf.WriteString(s)305
l.mu.Unlock()306
}307
308
func (l *logBuffer) take() string {309
l.mu.Lock()310
defer l.mu.Unlock()311
s := l.buf.String()312
l.buf.Reset()313
return s314
}315
316
func (c *client) runJob(parent context.Context, pl runner.RemotePayload) {317
c.log.Info("job", "id", pl.JobID, "name", pl.Job.Name)318
base := fmt.Sprintf("/api/v1/runner/jobs/%d", pl.JobID)319
ctx, cancel := context.WithCancel(parent)320
defer cancel()321
runCtx := ctx322
if pl.Timeout > 0 {323
var tc context.CancelFunc324
runCtx, tc = context.WithTimeout(ctx, time.Duration(pl.Timeout)*time.Second)325
defer tc()326
}327
var canceled bool328
var cmu sync.Mutex329
logs := &logBuffer{}330
send := func() {331
chunk := logs.take()332
var out struct {333
Cancel bool `json:"cancel"`334
}335
sctx, sc := context.WithTimeout(context.Background(), 30*time.Second)336
defer sc()337
err := func() error {338
resp, err := c.do(sctx, http.MethodPost, base+"/log", strings.NewReader(chunk), "text/plain")339
if err != nil {340
return err341
}342
defer resp.Body.Close()343
if resp.StatusCode != http.StatusOK {344
return fmt.Errorf("log: %d", resp.StatusCode)345
}346
return json.NewDecoder(resp.Body).Decode(&out)347
}()348
if err != nil {349
// Bij een fout gaat de chunk terug in de buffer voor de volgende poging.350
if chunk != "" {351
logs.mu.Lock()352
rest := logs.buf.String()353
logs.buf.Reset()354
logs.buf.WriteString(chunk + rest)355
logs.mu.Unlock()356
}357
return358
}359
if out.Cancel {360
cmu.Lock()361
canceled = true362
cmu.Unlock()363
cancel()364
}365
}366
done := make(chan struct{})367
flushed := make(chan struct{})368
go func() {369
defer close(flushed)370
t := time.NewTicker(time.Second)371
defer t.Stop()372
for {373
select {374
case <-done:375
return376
case <-t.C:377
send()378
}379
}380
}()381
r := &runner.Runner{GitBin: c.cfg.Git, Docker: c.cfg.Docker, Storage: c.cfg.WorkDir, Network: c.cfg.Network, Name: c.cfg.Name}382
work := filepath.Join(c.cfg.WorkDir, "work", strconv.FormatInt(pl.JobID, 10))383
defer os.RemoveAll(work)384
clone := func(ctx context.Context, dir string) error {385
cmd := exec.CommandContext(ctx, c.cfg.Git, "clone", "--quiet", pl.CloneURL, dir)386
// Het runner-token via de omgeving, niet in de proceslijst.387
cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_COUNT=1",388
"GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=Authorization: Bearer "+c.cfg.Token)389
out, err := cmd.CombinedOutput()390
if err != nil {391
return fmt.Errorf("%s", strings.ReplaceAll(strings.TrimSpace(string(out)), c.cfg.Token, "[MASKED]"))392
}393
return nil394
}395
inputs := func(ctx context.Context, dir string) error {396
resp, err := c.do(ctx, http.MethodGet, base+"/inputs", nil, "")397
if err != nil {398
return err399
}400
defer resp.Body.Close()401
if resp.StatusCode != http.StatusOK {402
return fmt.Errorf("inputs: %d", resp.StatusCode)403
}404
return runner.ReadTar(resp.Body, dir, nil, 0)405
}406
err := r.RunRemote(runCtx, pl, work, logs.add, clone, inputs)407
if err == nil && len(pl.Job.ArtifactPaths) > 0 {408
err = c.upload(runCtx, base, work, pl.Job.ArtifactPaths)409
}410
close(done)411
<-flushed412
send()413
status, msg := "success", ""414
cmu.Lock()415
wasCanceled := canceled416
cmu.Unlock()417
switch {418
case err == nil:419
case wasCanceled:420
status = "canceled"421
case errors.Is(runCtx.Err(), context.DeadlineExceeded):422
status, msg = "failed", "job_execution_timeout: "+err.Error()423
default:424
status, msg = "failed", err.Error()425
}426
fctx, fc := context.WithTimeout(context.Background(), time.Minute)427
defer fc()428
if ferr := c.postJSON(fctx, base+"/finish", map[string]string{"status": status, "message": msg}, nil); ferr != nil {429
c.log.Error("finish", "job", pl.JobID, "err", ferr)430
}431
c.log.Info("job done", "id", pl.JobID, "status", status)432
}433
434
func (c *client) upload(ctx context.Context, base, work string, paths []string) error {435
pr, pw := io.Pipe()436
go func() { pw.CloseWithError(runner.WriteTar(pw, work, paths)) }()437
resp, err := c.do(ctx, http.MethodPut, base+"/artifacts", pr, "application/gzip")438
if err != nil {439
return fmt.Errorf("artifacts upload: %w", err)440
}441
defer resp.Body.Close()442
if resp.StatusCode != http.StatusOK {443
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))444
return fmt.Errorf("artifacts upload: %d %s", resp.StatusCode, strings.TrimSpace(string(msg)))445
}446
return nil447
}@@ -53,7 +53,7 @@ jobs:
53
53
| `services` | Sidecar-containers op een eigen Docker-netwerk met de job. Een tekst, of `name`, `alias`, `command`, `entrypoint` en `variables`. Zonder alias is dat het imagenaam zonder registry en tag. |
54
54
| `environment` | Een naam, of `{name, url}`. Zet `CI_ENVIRONMENT_NAME` en `CI_ENVIRONMENT_URL`. Geen deployment en geen `on_stop`. |
55
55
| `interruptible` | Standaard uit. Een nieuwe pipeline op dezelfde ref annuleert oudere jobs die dit aan hebben en nog in de wachtrij staan of lopen. |
56
| `tags` | Bewaard bij de job. De ingebouwde runner draait elke job; er is geen tweede runner om op tags te kiezen. |
56
| `tags` | Bewaard bij de job. Tags kiezen geen runner; welke runner een job krijgt staat in [runner.md](runner.md). |
57
57
| `extends` | Eén naam of een lijst. Een job waarvan de naam met `.` begint is een sjabloon en draait niet. |
58
58
59
59
Een `if` volgt de GitLab-vorm. `&&` bindt sterker dan `||`. Haakjes groeperen. Een lege variabele is `null`.
@@ -53,7 +53,7 @@ jobs:53
| `services` | Sidecar-containers op een eigen Docker-netwerk met de job. Een tekst, of `name`, `alias`, `command`, `entrypoint` en `variables`. Zonder alias is dat het imagenaam zonder registry en tag. |53
| `services` | Sidecar-containers op een eigen Docker-netwerk met de job. Een tekst, of `name`, `alias`, `command`, `entrypoint` en `variables`. Zonder alias is dat het imagenaam zonder registry en tag. |54
| `environment` | Een naam, of `{name, url}`. Zet `CI_ENVIRONMENT_NAME` en `CI_ENVIRONMENT_URL`. Geen deployment en geen `on_stop`. |54
| `environment` | Een naam, of `{name, url}`. Zet `CI_ENVIRONMENT_NAME` en `CI_ENVIRONMENT_URL`. Geen deployment en geen `on_stop`. |55
| `interruptible` | Standaard uit. Een nieuwe pipeline op dezelfde ref annuleert oudere jobs die dit aan hebben en nog in de wachtrij staan of lopen. |55
| `interruptible` | Standaard uit. Een nieuwe pipeline op dezelfde ref annuleert oudere jobs die dit aan hebben en nog in de wachtrij staan of lopen. |56
| `tags` | Bewaard bij de job. De ingebouwde runner draait elke job; er is geen tweede runner om op tags te kiezen. |56
| `tags` | Bewaard bij de job. Tags kiezen geen runner; welke runner een job krijgt staat in [runner.md](runner.md). |57
| `extends` | Eén naam of een lijst. Een job waarvan de naam met `.` begint is een sjabloon en draait niet. |57
| `extends` | Eén naam of een lijst. Een job waarvan de naam met `.` begint is een sjabloon en draait niet. |58
58
59
Een `if` volgt de GitLab-vorm. `&&` bindt sterker dan `||`. Haakjes groeperen. Een lege variabele is `null`.59
Een `if` volgt de GitLab-vorm. `&&` bindt sterker dan `||`. Haakjes groeperen. Een lege variabele is `null`.@@ -0,0 +1,106 @@
1
# bluegit-runner — aparte CI-runner
2
3
`bluegit-runner` draait BlueGit CI-jobs op je eigen machine, in Docker,
4
net als de ingebouwde runner. De runner maakt alleen **uitgaande**
5
HTTPS-verbindingen naar BlueGit (long-poll); er hoeft geen poort open.
6
De ingebouwde BlueGit-runner blijft gewoon bestaan.
7
8
## Installeren (Arch / AUR)
9
10
```sh
11
yay -S bluegit-runner # of: git clone https://aur.archlinux.org/bluegit-runner.git && makepkg -si
12
```
13
14
Het pakket maakt de systeemgebruiker `bluegit-runner` (lid van `docker`),
15
installeert `bluegit-runner.service` en zet die aan. De service start
16
zodra `/etc/bluegit-runner/config.yml` bestaat; een upgrade herstart hem.
17
18
Andere systemen: `go build -o /usr/bin/bluegit-runner ./cmd/bluegit-runner`
19
en kopieer de bestanden uit `packaging/runner/` naar systemd
20
(`bluegit-runner.service`, `sysusers.d`, `tmpfiles.d`).
21
22
## Registreren
23
24
1. In BlueGit: **Instellingen → Runners → Nieuw registratietoken** voor een
25
account-runner, of **Groep → Instellingen → Runners** voor een
26
groepsrunner (maintainer). Het token (`bgrr_…`) is 24 uur geldig,
27
eenmalig, en je ziet het één keer.
28
2. Op de machine:
29
30
```sh
31
sudo bluegit-runner register --url https://git.timfalken.com --token bgrr_... [--name mijn-pc]
32
```
33
34
De runner krijgt een eigen runner-token (`bgr_…`); de server bewaart
35
alleen de hash. Het token staat in `/etc/bluegit-runner/config.yml`
36
(mode 0600, eigenaar `bluegit-runner`). `register` start de service.
37
3. In de UI staat de runner nu als **online**. Intrekken kan daar ook;
38
het token werkt dan meteen niet meer.
39
40
Het registratietoken mag ook via `BLUEGIT_RUNNER_REGISTRATION_TOKEN`.
41
42
## Config
43
44
`/etc/bluegit-runner/config.yml`:
45
46
| sleutel | standaard | |
47
|---|---|---|
48
| `url` | — | BlueGit-adres (door register) |
49
| `token` | — | runner-token (door register) |
50
| `name` | hostname | naam in de UI en in de joblog |
51
| `docker` | `docker` | docker-binary |
52
| `git` | `git` | git-binary |
53
| `work_dir` | `/var/lib/bluegit-runner` | werkmappen en cache |
54
| `network` | `bridge` | docker-netwerk voor jobs zonder services |
55
| `concurrency` | `1` | jobs tegelijk |
56
57
## Welke runner krijgt een job?
58
59
**Repo van een account** — voorkeur onder Instellingen → Runners:
60
61
- *Mijn runners alleen als de BlueGit-runner niet meteen beschikbaar is*
62
(standaard): BlueGit-runner → eigen runners.
63
- *Eigen runners eerst*: eigen runners → BlueGit-runner.
64
65
**Repo van een groep**: groepsrunners → account-runners van maintainers
66
die toestemming gaven (Groep → Instellingen → Runners → "Mijn runners
67
toestaan") → BlueGit-runner. Met **Probeer BlueGit runner eerst** staat
68
de BlueGit-runner vooraan.
69
70
Een trede zonder online runner wordt meteen overgeslagen. Pakt geen
71
runner de job binnen 20 seconden op, of is de BlueGit-runner vol, dan
72
gaat de job naar de volgende trede (en daarna weer rond). Een runner die
73
midden in een job 5 minuten niets van zich laat horen, laat de job
74
mislukken met `runner_system_failure` (retry-regels gelden).
75
76
**Privacy**: een account-runner krijgt alleen jobs uit repo's die zijn
77
eigenaar mag lezen; een groepsrunner alleen uit repo's van die groep.
78
Toestemming van een maintainer telt alleen zolang die maintainer is.
79
Het runner-token mag alleen de repo clonen van een job die die runner op
80
dat moment draait. De rechten van `CI_JOB_TOKEN` zijn ongewijzigd.
81
82
## Wat gebeurt er per job
83
84
1. Long-poll `POST /api/v1/runner/jobs/request` geeft de job: script,
85
variabelen (incl. geheimen en `CI_JOB_TOKEN`), maskers, clone-URL, sha.
86
2. Clone via HTTPS met het runner-token, checkout van de sha.
87
3. Artifacts van eerdere jobs: `GET /api/v1/runner/jobs/{id}/inputs`.
88
4. Container draaien (zelfde code als de ingebouwde runner, incl. image
89
presets en services); log elke seconde naar
90
`POST /api/v1/runner/jobs/{id}/log` (antwoord `{"cancel":true}` stopt de job).
91
5. Artifacts: `PUT /api/v1/runner/jobs/{id}/artifacts` (alleen de gedeclareerde paden).
92
6. `POST /api/v1/runner/jobs/{id}/finish`.
93
94
Heartbeat: `POST /api/v1/runner/heartbeat` elke 30 s; online = contact in
95
de laatste 90 s. De **cache** (`cache:`) blijft lokaal op de
96
runnermachine in `work_dir/cache`.
97
98
De job-UI en de API (`runner` in job-JSON: `{id, name, kind}`) tonen
99
welke runner een job draaide.
100
101
## Problemen
102
103
```sh
104
journalctl -u bluegit-runner -f
105
sudo -u bluegit-runner docker info # moet werken (groep docker)
106
```
@@ -0,0 +1,106 @@1
# bluegit-runner — aparte CI-runner2
3
`bluegit-runner` draait BlueGit CI-jobs op je eigen machine, in Docker,4
net als de ingebouwde runner. De runner maakt alleen **uitgaande**5
HTTPS-verbindingen naar BlueGit (long-poll); er hoeft geen poort open.6
De ingebouwde BlueGit-runner blijft gewoon bestaan.7
8
## Installeren (Arch / AUR)9
10
```sh11
yay -S bluegit-runner # of: git clone https://aur.archlinux.org/bluegit-runner.git && makepkg -si12
```13
14
Het pakket maakt de systeemgebruiker `bluegit-runner` (lid van `docker`),15
installeert `bluegit-runner.service` en zet die aan. De service start16
zodra `/etc/bluegit-runner/config.yml` bestaat; een upgrade herstart hem.17
18
Andere systemen: `go build -o /usr/bin/bluegit-runner ./cmd/bluegit-runner`19
en kopieer de bestanden uit `packaging/runner/` naar systemd20
(`bluegit-runner.service`, `sysusers.d`, `tmpfiles.d`).21
22
## Registreren23
24
1. In BlueGit: **Instellingen → Runners → Nieuw registratietoken** voor een25
account-runner, of **Groep → Instellingen → Runners** voor een26
groepsrunner (maintainer). Het token (`bgrr_…`) is 24 uur geldig,27
eenmalig, en je ziet het één keer.28
2. Op de machine:29
30
```sh31
sudo bluegit-runner register --url https://git.timfalken.com --token bgrr_... [--name mijn-pc]32
```33
34
De runner krijgt een eigen runner-token (`bgr_…`); de server bewaart35
alleen de hash. Het token staat in `/etc/bluegit-runner/config.yml`36
(mode 0600, eigenaar `bluegit-runner`). `register` start de service.37
3. In de UI staat de runner nu als **online**. Intrekken kan daar ook;38
het token werkt dan meteen niet meer.39
40
Het registratietoken mag ook via `BLUEGIT_RUNNER_REGISTRATION_TOKEN`.41
42
## Config43
44
`/etc/bluegit-runner/config.yml`:45
46
| sleutel | standaard | |47
|---|---|---|48
| `url` | — | BlueGit-adres (door register) |49
| `token` | — | runner-token (door register) |50
| `name` | hostname | naam in de UI en in de joblog |51
| `docker` | `docker` | docker-binary |52
| `git` | `git` | git-binary |53
| `work_dir` | `/var/lib/bluegit-runner` | werkmappen en cache |54
| `network` | `bridge` | docker-netwerk voor jobs zonder services |55
| `concurrency` | `1` | jobs tegelijk |56
57
## Welke runner krijgt een job?58
59
**Repo van een account** — voorkeur onder Instellingen → Runners:60
61
- *Mijn runners alleen als de BlueGit-runner niet meteen beschikbaar is*62
(standaard): BlueGit-runner → eigen runners.63
- *Eigen runners eerst*: eigen runners → BlueGit-runner.64
65
**Repo van een groep**: groepsrunners → account-runners van maintainers66
die toestemming gaven (Groep → Instellingen → Runners → "Mijn runners67
toestaan") → BlueGit-runner. Met **Probeer BlueGit runner eerst** staat68
de BlueGit-runner vooraan.69
70
Een trede zonder online runner wordt meteen overgeslagen. Pakt geen71
runner de job binnen 20 seconden op, of is de BlueGit-runner vol, dan72
gaat de job naar de volgende trede (en daarna weer rond). Een runner die73
midden in een job 5 minuten niets van zich laat horen, laat de job74
mislukken met `runner_system_failure` (retry-regels gelden).75
76
**Privacy**: een account-runner krijgt alleen jobs uit repo's die zijn77
eigenaar mag lezen; een groepsrunner alleen uit repo's van die groep.78
Toestemming van een maintainer telt alleen zolang die maintainer is.79
Het runner-token mag alleen de repo clonen van een job die die runner op80
dat moment draait. De rechten van `CI_JOB_TOKEN` zijn ongewijzigd.81
82
## Wat gebeurt er per job83
84
1. Long-poll `POST /api/v1/runner/jobs/request` geeft de job: script,85
variabelen (incl. geheimen en `CI_JOB_TOKEN`), maskers, clone-URL, sha.86
2. Clone via HTTPS met het runner-token, checkout van de sha.87
3. Artifacts van eerdere jobs: `GET /api/v1/runner/jobs/{id}/inputs`.88
4. Container draaien (zelfde code als de ingebouwde runner, incl. image89
presets en services); log elke seconde naar90
`POST /api/v1/runner/jobs/{id}/log` (antwoord `{"cancel":true}` stopt de job).91
5. Artifacts: `PUT /api/v1/runner/jobs/{id}/artifacts` (alleen de gedeclareerde paden).92
6. `POST /api/v1/runner/jobs/{id}/finish`.93
94
Heartbeat: `POST /api/v1/runner/heartbeat` elke 30 s; online = contact in95
de laatste 90 s. De **cache** (`cache:`) blijft lokaal op de96
runnermachine in `work_dir/cache`.97
98
De job-UI en de API (`runner` in job-JSON: `{id, name, kind}`) tonen99
welke runner een job draaide.100
101
## Problemen102
103
```sh104
journalctl -u bluegit-runner -f105
sudo -u bluegit-runner docker info # moet werken (groep docker)106
```@@ -19,7 +19,7 @@ func TestBlueGitExampleFile(t *testing.T) {
19
19
if len(p.Warnings) != 0 {
20
20
t.Fatalf("warnings %v", p.Warnings)
21
21
}
22
if len(p.Jobs) != 6 {
22
if len(p.Jobs) != 7 {
23
23
t.Fatalf("jobs %d", len(p.Jobs))
24
24
}
25
25
by := map[string]Job{}
@@ -77,14 +77,14 @@ func TestBlueGitExampleFile(t *testing.T) {
77
77
web, err := p.ActiveJobs(map[string]string{
78
78
"CI_COMMIT_BRANCH": "master", "CI_PIPELINE_SOURCE": "web",
79
79
})
80
if err != nil || len(web) != 6 {
80
if err != nil || len(web) != 7 {
81
81
t.Fatalf("web op master %d %v", len(web), err)
82
82
}
83
83
84
84
active, err := p.ActiveJobs(map[string]string{
85
85
"CI_COMMIT_BRANCH": "master", "CI_PIPELINE_SOURCE": "push",
86
86
})
87
if err != nil || len(active) != 6 {
87
if err != nil || len(active) != 7 {
88
88
t.Fatalf("master %d %v", len(active), err)
89
89
}
90
90
aur := by["publish_aur"]
@@ -107,8 +107,13 @@ func TestBlueGitExampleFile(t *testing.T) {
107
107
}, aur) != "ready" {
108
108
t.Fatal("publish_aur start na build, vóór self_update")
109
109
}
110
runnerAUR := by["publish_aur_runner"]
111
if !runnerAUR.AllowFailure || len(runnerAUR.Needs) != 1 || runnerAUR.Needs[0] != "build" ||
112
!strings.Contains(strings.Join(runnerAUR.Script, "\n"), "AUR_PKG=bluegit-runner ./scripts/publish-aur.sh") {
113
t.Fatalf("publish_aur_runner: %+v", runnerAUR)
114
}
110
115
upd := by["self_update"]
111
if !upd.NeedsSet || len(upd.Needs) != 1 || upd.Needs[0] != "publish_aur" {
116
if !upd.NeedsSet || len(upd.Needs) != 2 || upd.Needs[0] != "publish_aur" || upd.Needs[1] != "publish_aur_runner" {
112
117
t.Fatalf("self_update hoort op publish_aur te wachten: %+v", upd)
113
118
}
114
119
joined := strings.Join(upd.Script, "\n")
@@ -139,13 +144,13 @@ func TestBlueGitExampleFile(t *testing.T) {
139
144
}
140
145
if Gate(p.Jobs, map[string]string{
141
146
"vet": "success", "test": "success", "frontend_checks": "success",
142
"build": "success", "publish_aur": "success",
147
"build": "success", "publish_aur": "success", "publish_aur_runner": "success",
143
148
}, upd) != "ready" {
144
149
t.Fatal("self_update start na een geslaagde publish_aur")
145
150
}
146
151
if Gate(p.Jobs, map[string]string{
147
152
"vet": "success", "test": "success", "frontend_checks": "success",
148
"build": "success", "publish_aur": "failed",
153
"build": "success", "publish_aur": "failed", "publish_aur_runner": "failed",
149
154
}, upd) != "ready" {
150
155
t.Fatal("een AUR-fout met allow_failure hoort de deploy niet te blokkeren")
151
156
}
@@ -19,7 +19,7 @@ func TestBlueGitExampleFile(t *testing.T) {19
if len(p.Warnings) != 0 {19
if len(p.Warnings) != 0 {20
t.Fatalf("warnings %v", p.Warnings)20
t.Fatalf("warnings %v", p.Warnings)21
}21
}22
if len(p.Jobs) != 6 {22
if len(p.Jobs) != 7 {23
t.Fatalf("jobs %d", len(p.Jobs))23
t.Fatalf("jobs %d", len(p.Jobs))24
}24
}25
by := map[string]Job{}25
by := map[string]Job{}@@ -77,14 +77,14 @@ func TestBlueGitExampleFile(t *testing.T) {77
web, err := p.ActiveJobs(map[string]string{77
web, err := p.ActiveJobs(map[string]string{78
"CI_COMMIT_BRANCH": "master", "CI_PIPELINE_SOURCE": "web",78
"CI_COMMIT_BRANCH": "master", "CI_PIPELINE_SOURCE": "web",79
})79
})80
if err != nil || len(web) != 6 {80
if err != nil || len(web) != 7 {81
t.Fatalf("web op master %d %v", len(web), err)81
t.Fatalf("web op master %d %v", len(web), err)82
}82
}83
83
84
active, err := p.ActiveJobs(map[string]string{84
active, err := p.ActiveJobs(map[string]string{85
"CI_COMMIT_BRANCH": "master", "CI_PIPELINE_SOURCE": "push",85
"CI_COMMIT_BRANCH": "master", "CI_PIPELINE_SOURCE": "push",86
})86
})87
if err != nil || len(active) != 6 {87
if err != nil || len(active) != 7 {88
t.Fatalf("master %d %v", len(active), err)88
t.Fatalf("master %d %v", len(active), err)89
}89
}90
aur := by["publish_aur"]90
aur := by["publish_aur"]@@ -107,8 +107,13 @@ func TestBlueGitExampleFile(t *testing.T) {107
}, aur) != "ready" {107
}, aur) != "ready" {108
t.Fatal("publish_aur start na build, vóór self_update")108
t.Fatal("publish_aur start na build, vóór self_update")109
}109
}110
runnerAUR := by["publish_aur_runner"]111
if !runnerAUR.AllowFailure || len(runnerAUR.Needs) != 1 || runnerAUR.Needs[0] != "build" ||112
!strings.Contains(strings.Join(runnerAUR.Script, "\n"), "AUR_PKG=bluegit-runner ./scripts/publish-aur.sh") {113
t.Fatalf("publish_aur_runner: %+v", runnerAUR)114
}110
upd := by["self_update"]115
upd := by["self_update"]111
if !upd.NeedsSet || len(upd.Needs) != 1 || upd.Needs[0] != "publish_aur" {116
if !upd.NeedsSet || len(upd.Needs) != 2 || upd.Needs[0] != "publish_aur" || upd.Needs[1] != "publish_aur_runner" {112
t.Fatalf("self_update hoort op publish_aur te wachten: %+v", upd)117
t.Fatalf("self_update hoort op publish_aur te wachten: %+v", upd)113
}118
}114
joined := strings.Join(upd.Script, "\n")119
joined := strings.Join(upd.Script, "\n")@@ -139,13 +144,13 @@ func TestBlueGitExampleFile(t *testing.T) {139
}144
}140
if Gate(p.Jobs, map[string]string{145
if Gate(p.Jobs, map[string]string{141
"vet": "success", "test": "success", "frontend_checks": "success",146
"vet": "success", "test": "success", "frontend_checks": "success",142
"build": "success", "publish_aur": "success",147
"build": "success", "publish_aur": "success", "publish_aur_runner": "success",143
}, upd) != "ready" {148
}, upd) != "ready" {144
t.Fatal("self_update start na een geslaagde publish_aur")149
t.Fatal("self_update start na een geslaagde publish_aur")145
}150
}146
if Gate(p.Jobs, map[string]string{151
if Gate(p.Jobs, map[string]string{147
"vet": "success", "test": "success", "frontend_checks": "success",152
"vet": "success", "test": "success", "frontend_checks": "success",148
"build": "success", "publish_aur": "failed",153
"build": "success", "publish_aur": "failed", "publish_aur_runner": "failed",149
}, upd) != "ready" {154
}, upd) != "ready" {150
t.Fatal("een AUR-fout met allow_failure hoort de deploy niet te blokkeren")155
t.Fatal("een AUR-fout met allow_failure hoort de deploy niet te blokkeren")151
}156
}@@ -0,0 +1,112 @@
1
package i18n
2
3
func init() {
4
addCatalog(en, runnerEN)
5
addCatalog(nl, runnerNL)
6
addCatalog(de, runnerDE)
7
addCatalog(fr, runnerFR)
8
}
9
10
var runnerEN = map[string]string{
11
"runner.title": "Runners",
12
"runner.userLead": "Run CI jobs of your projects on your own machine with bluegit-runner. Create a registration token and run the command on that machine.",
13
"runner.groupLead": "Order for this group: group runners, then account runners of maintainers who allowed it, then the BlueGit runner.",
14
"runner.prefBuiltin": "Use my runners only when the BlueGit runner is not available right away",
15
"runner.prefOwn": "My own runners first",
16
"runner.builtinFirst": "Try the BlueGit runner first",
17
"runner.optinOff": "Your account runners are not used for this group.",
18
"runner.optinOn": "Your account runners may run jobs of this group.",
19
"runner.optinStart": "Allow my runners for this group",
20
"runner.optinStop": "Stop using my runners here",
21
"runner.optinWho": "Allowed by:",
22
"runner.new": "New registration token",
23
"runner.tokenOnce": "Registration token (valid 24 hours, single use). It is shown only once:",
24
"runner.tokenMade": "Registration token created.",
25
"runner.revoked": "Runner revoked.",
26
"runner.missing": "That runner does not exist.",
27
"runner.revokeAsk": "Revoke this runner? It can no longer pick up jobs.",
28
"runner.none": "No runners registered yet.",
29
"runner.status": "Status",
30
"runner.seen": "Last contact",
31
"runner.version": "Version",
32
"runner.online": "online",
33
"runner.offline": "offline",
34
}
35
36
var runnerNL = map[string]string{
37
"runner.title": "Runners",
38
"runner.userLead": "Draai CI-jobs van je projecten op je eigen machine met bluegit-runner. Maak een registratietoken en voer het commando uit op die machine.",
39
"runner.groupLead": "Volgorde voor deze groep: groepsrunners, dan account-runners van maintainers die toestemming gaven, dan de BlueGit-runner.",
40
"runner.prefBuiltin": "Mijn runners alleen als de BlueGit-runner niet meteen beschikbaar is",
41
"runner.prefOwn": "Eigen runners eerst",
42
"runner.builtinFirst": "Probeer BlueGit runner eerst",
43
"runner.optinOff": "Je account-runners worden niet gebruikt voor deze groep.",
44
"runner.optinOn": "Je account-runners mogen jobs van deze groep draaien.",
45
"runner.optinStart": "Mijn runners toestaan voor deze groep",
46
"runner.optinStop": "Mijn runners hier niet meer gebruiken",
47
"runner.optinWho": "Toegestaan door:",
48
"runner.new": "Nieuw registratietoken",
49
"runner.tokenOnce": "Registratietoken (24 uur geldig, eenmalig). Je ziet het maar één keer:",
50
"runner.tokenMade": "Registratietoken gemaakt.",
51
"runner.revoked": "Runner ingetrokken.",
52
"runner.missing": "Die runner bestaat niet.",
53
"runner.revokeAsk": "Deze runner intrekken? Hij kan dan geen jobs meer oppakken.",
54
"runner.none": "Nog geen runners geregistreerd.",
55
"runner.status": "Status",
56
"runner.seen": "Laatste contact",
57
"runner.version": "Versie",
58
"runner.online": "online",
59
"runner.offline": "offline",
60
}
61
62
var runnerDE = map[string]string{
63
"runner.title": "Runner",
64
"runner.userLead": "Führe CI-Jobs deiner Projekte mit bluegit-runner auf deinem eigenen Rechner aus. Erstelle ein Registrierungstoken und führe den Befehl dort aus.",
65
"runner.groupLead": "Reihenfolge für diese Gruppe: Gruppen-Runner, dann Konto-Runner von Maintainern mit Zustimmung, dann der BlueGit-Runner.",
66
"runner.prefBuiltin": "Meine Runner nur, wenn der BlueGit-Runner nicht sofort frei ist",
67
"runner.prefOwn": "Eigene Runner zuerst",
68
"runner.builtinFirst": "Zuerst den BlueGit-Runner versuchen",
69
"runner.optinOff": "Deine Konto-Runner werden für diese Gruppe nicht verwendet.",
70
"runner.optinOn": "Deine Konto-Runner dürfen Jobs dieser Gruppe ausführen.",
71
"runner.optinStart": "Meine Runner für diese Gruppe erlauben",
72
"runner.optinStop": "Meine Runner hier nicht mehr verwenden",
73
"runner.optinWho": "Erlaubt von:",
74
"runner.new": "Neues Registrierungstoken",
75
"runner.tokenOnce": "Registrierungstoken (24 Stunden gültig, einmalig). Es wird nur einmal angezeigt:",
76
"runner.tokenMade": "Registrierungstoken erstellt.",
77
"runner.revoked": "Runner widerrufen.",
78
"runner.missing": "Dieser Runner existiert nicht.",
79
"runner.revokeAsk": "Diesen Runner widerrufen? Er kann dann keine Jobs mehr annehmen.",
80
"runner.none": "Noch keine Runner registriert.",
81
"runner.status": "Status",
82
"runner.seen": "Letzter Kontakt",
83
"runner.version": "Version",
84
"runner.online": "online",
85
"runner.offline": "offline",
86
}
87
88
var runnerFR = map[string]string{
89
"runner.title": "Runners",
90
"runner.userLead": "Exécutez les jobs CI de vos projets sur votre propre machine avec bluegit-runner. Créez un jeton d'enregistrement et lancez la commande sur cette machine.",
91
"runner.groupLead": "Ordre pour ce groupe : runners du groupe, puis runners de compte des mainteneurs qui l'ont autorisé, puis le runner BlueGit.",
92
"runner.prefBuiltin": "Mes runners seulement si le runner BlueGit n'est pas disponible tout de suite",
93
"runner.prefOwn": "Mes runners d'abord",
94
"runner.builtinFirst": "Essayer d'abord le runner BlueGit",
95
"runner.optinOff": "Vos runners de compte ne sont pas utilisés pour ce groupe.",
96
"runner.optinOn": "Vos runners de compte peuvent exécuter les jobs de ce groupe.",
97
"runner.optinStart": "Autoriser mes runners pour ce groupe",
98
"runner.optinStop": "Ne plus utiliser mes runners ici",
99
"runner.optinWho": "Autorisé par :",
100
"runner.new": "Nouveau jeton d'enregistrement",
101
"runner.tokenOnce": "Jeton d'enregistrement (valable 24 heures, usage unique). Il n'est affiché qu'une fois :",
102
"runner.tokenMade": "Jeton d'enregistrement créé.",
103
"runner.revoked": "Runner révoqué.",
104
"runner.missing": "Ce runner n'existe pas.",
105
"runner.revokeAsk": "Révoquer ce runner ? Il ne pourra plus prendre de jobs.",
106
"runner.none": "Aucun runner enregistré.",
107
"runner.status": "Statut",
108
"runner.seen": "Dernier contact",
109
"runner.version": "Version",
110
"runner.online": "en ligne",
111
"runner.offline": "hors ligne",
112
}
@@ -0,0 +1,112 @@1
package i18n2
3
func init() {4
addCatalog(en, runnerEN)5
addCatalog(nl, runnerNL)6
addCatalog(de, runnerDE)7
addCatalog(fr, runnerFR)8
}9
10
var runnerEN = map[string]string{11
"runner.title": "Runners",12
"runner.userLead": "Run CI jobs of your projects on your own machine with bluegit-runner. Create a registration token and run the command on that machine.",13
"runner.groupLead": "Order for this group: group runners, then account runners of maintainers who allowed it, then the BlueGit runner.",14
"runner.prefBuiltin": "Use my runners only when the BlueGit runner is not available right away",15
"runner.prefOwn": "My own runners first",16
"runner.builtinFirst": "Try the BlueGit runner first",17
"runner.optinOff": "Your account runners are not used for this group.",18
"runner.optinOn": "Your account runners may run jobs of this group.",19
"runner.optinStart": "Allow my runners for this group",20
"runner.optinStop": "Stop using my runners here",21
"runner.optinWho": "Allowed by:",22
"runner.new": "New registration token",23
"runner.tokenOnce": "Registration token (valid 24 hours, single use). It is shown only once:",24
"runner.tokenMade": "Registration token created.",25
"runner.revoked": "Runner revoked.",26
"runner.missing": "That runner does not exist.",27
"runner.revokeAsk": "Revoke this runner? It can no longer pick up jobs.",28
"runner.none": "No runners registered yet.",29
"runner.status": "Status",30
"runner.seen": "Last contact",31
"runner.version": "Version",32
"runner.online": "online",33
"runner.offline": "offline",34
}35
36
var runnerNL = map[string]string{37
"runner.title": "Runners",38
"runner.userLead": "Draai CI-jobs van je projecten op je eigen machine met bluegit-runner. Maak een registratietoken en voer het commando uit op die machine.",39
"runner.groupLead": "Volgorde voor deze groep: groepsrunners, dan account-runners van maintainers die toestemming gaven, dan de BlueGit-runner.",40
"runner.prefBuiltin": "Mijn runners alleen als de BlueGit-runner niet meteen beschikbaar is",41
"runner.prefOwn": "Eigen runners eerst",42
"runner.builtinFirst": "Probeer BlueGit runner eerst",43
"runner.optinOff": "Je account-runners worden niet gebruikt voor deze groep.",44
"runner.optinOn": "Je account-runners mogen jobs van deze groep draaien.",45
"runner.optinStart": "Mijn runners toestaan voor deze groep",46
"runner.optinStop": "Mijn runners hier niet meer gebruiken",47
"runner.optinWho": "Toegestaan door:",48
"runner.new": "Nieuw registratietoken",49
"runner.tokenOnce": "Registratietoken (24 uur geldig, eenmalig). Je ziet het maar één keer:",50
"runner.tokenMade": "Registratietoken gemaakt.",51
"runner.revoked": "Runner ingetrokken.",52
"runner.missing": "Die runner bestaat niet.",53
"runner.revokeAsk": "Deze runner intrekken? Hij kan dan geen jobs meer oppakken.",54
"runner.none": "Nog geen runners geregistreerd.",55
"runner.status": "Status",56
"runner.seen": "Laatste contact",57
"runner.version": "Versie",58
"runner.online": "online",59
"runner.offline": "offline",60
}61
62
var runnerDE = map[string]string{63
"runner.title": "Runner",64
"runner.userLead": "Führe CI-Jobs deiner Projekte mit bluegit-runner auf deinem eigenen Rechner aus. Erstelle ein Registrierungstoken und führe den Befehl dort aus.",65
"runner.groupLead": "Reihenfolge für diese Gruppe: Gruppen-Runner, dann Konto-Runner von Maintainern mit Zustimmung, dann der BlueGit-Runner.",66
"runner.prefBuiltin": "Meine Runner nur, wenn der BlueGit-Runner nicht sofort frei ist",67
"runner.prefOwn": "Eigene Runner zuerst",68
"runner.builtinFirst": "Zuerst den BlueGit-Runner versuchen",69
"runner.optinOff": "Deine Konto-Runner werden für diese Gruppe nicht verwendet.",70
"runner.optinOn": "Deine Konto-Runner dürfen Jobs dieser Gruppe ausführen.",71
"runner.optinStart": "Meine Runner für diese Gruppe erlauben",72
"runner.optinStop": "Meine Runner hier nicht mehr verwenden",73
"runner.optinWho": "Erlaubt von:",74
"runner.new": "Neues Registrierungstoken",75
"runner.tokenOnce": "Registrierungstoken (24 Stunden gültig, einmalig). Es wird nur einmal angezeigt:",76
"runner.tokenMade": "Registrierungstoken erstellt.",77
"runner.revoked": "Runner widerrufen.",78
"runner.missing": "Dieser Runner existiert nicht.",79
"runner.revokeAsk": "Diesen Runner widerrufen? Er kann dann keine Jobs mehr annehmen.",80
"runner.none": "Noch keine Runner registriert.",81
"runner.status": "Status",82
"runner.seen": "Letzter Kontakt",83
"runner.version": "Version",84
"runner.online": "online",85
"runner.offline": "offline",86
}87
88
var runnerFR = map[string]string{89
"runner.title": "Runners",90
"runner.userLead": "Exécutez les jobs CI de vos projets sur votre propre machine avec bluegit-runner. Créez un jeton d'enregistrement et lancez la commande sur cette machine.",91
"runner.groupLead": "Ordre pour ce groupe : runners du groupe, puis runners de compte des mainteneurs qui l'ont autorisé, puis le runner BlueGit.",92
"runner.prefBuiltin": "Mes runners seulement si le runner BlueGit n'est pas disponible tout de suite",93
"runner.prefOwn": "Mes runners d'abord",94
"runner.builtinFirst": "Essayer d'abord le runner BlueGit",95
"runner.optinOff": "Vos runners de compte ne sont pas utilisés pour ce groupe.",96
"runner.optinOn": "Vos runners de compte peuvent exécuter les jobs de ce groupe.",97
"runner.optinStart": "Autoriser mes runners pour ce groupe",98
"runner.optinStop": "Ne plus utiliser mes runners ici",99
"runner.optinWho": "Autorisé par :",100
"runner.new": "Nouveau jeton d'enregistrement",101
"runner.tokenOnce": "Jeton d'enregistrement (valable 24 heures, usage unique). Il n'est affiché qu'une fois :",102
"runner.tokenMade": "Jeton d'enregistrement créé.",103
"runner.revoked": "Runner révoqué.",104
"runner.missing": "Ce runner n'existe pas.",105
"runner.revokeAsk": "Révoquer ce runner ? Il ne pourra plus prendre de jobs.",106
"runner.none": "Aucun runner enregistré.",107
"runner.status": "Statut",108
"runner.seen": "Dernier contact",109
"runner.version": "Version",110
"runner.online": "en ligne",111
"runner.offline": "hors ligne",112
}@@ -0,0 +1,353 @@
1
package runner
2
3
import (
4
"archive/tar"
5
"compress/gzip"
6
"context"
7
"errors"
8
"fmt"
9
"io"
10
"os"
11
"path/filepath"
12
"strings"
13
"sync"
14
"time"
15
16
"gitlab.com/Timfa/bluegit/internal/ci"
17
"gitlab.com/Timfa/bluegit/internal/store"
18
)
19
20
// RemotePayload is wat een aparte runner bij een job krijgt.
21
type RemotePayload struct {
22
JobID int64 `json:"job_id"`
23
PipelineID int64 `json:"pipeline_id"`
24
RepoID int64 `json:"repo_id"`
25
Job store.Job `json:"job"`
26
CloneURL string `json:"clone_url"`
27
SHA string `json:"sha"`
28
Script string `json:"script"`
29
Env map[string]string `json:"env"`
30
Mask []string `json:"mask"`
31
Timeout int64 `json:"timeout_seconds"`
32
}
33
34
var remoteMasks sync.Map // job id → []string
35
36
// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.
37
var RemoteOnlineGrace = 5 * time.Minute
38
39
// Claim zoekt een aangeboden job voor runner rn en zet hem op running.
40
// nil, nil als er niets is.
41
func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {
42
if r.storage() == "" {
43
return nil, nil
44
}
45
jobs, runs, err := r.Store.OfferedJobs(ctx)
46
if err != nil {
47
return nil, err
48
}
49
cache := map[int64][]Tier{}
50
for _, j := range jobs {
51
tiers := r.tiersFor(ctx, cache, j.RepoID)
52
if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {
53
continue
54
}
55
repo, err := r.Store.RepoByID(ctx, j.RepoID)
56
if err != nil || !r.mayRun(ctx, rn, repo) {
57
continue
58
}
59
p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)
60
if err != nil || (r.Hold != nil && r.Hold(p)) {
61
continue
62
}
63
if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
64
continue
65
}
66
ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)
67
if err != nil || !ok {
68
continue
69
}
70
pl, err := r.payload(ctx, repo, p, j, rn)
71
if err != nil {
72
r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())
73
continue
74
}
75
return pl, nil
76
}
77
return nil, nil
78
}
79
80
// mayRun is de privacycontrole: een account-runner krijgt alleen jobs
81
// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit
82
// repo's van die groep.
83
func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool {
84
switch rn.OwnerKind {
85
case "group":
86
return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID
87
case "user":
88
if repo.OwnerKind == "user" {
89
return repo.OwnerID == rn.OwnerID
90
}
91
role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)
92
return err == nil && role.CanRead()
93
}
94
return false
95
}
96
97
func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {
98
token, err := r.Store.CreateJobToken(ctx, job.ID)
99
if err != nil {
100
return nil, err
101
}
102
env, mask, err := r.environment(ctx, repo, p, job, token)
103
if err != nil {
104
return nil, err
105
}
106
remoteMasks.Store(job.ID, mask)
107
limit := r.Timeout
108
if job.Timeout > 0 {
109
limit = job.Timeout
110
}
111
script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})
112
return &RemotePayload{
113
JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,
114
CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",
115
SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,
116
Timeout: int64(limit / time.Second),
117
}, nil
118
}
119
120
// RemoteLog voegt logregels van een aparte runner toe. cancel is true als
121
// de job niet meer loopt (geannuleerd of afgebroken).
122
func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {
123
if job.Status != "running" {
124
return true, nil
125
}
126
if m, ok := remoteMasks.Load(job.ID); ok {
127
chunk = ci.Mask(chunk, m.([]string))
128
}
129
if len(chunk) > 0 {
130
err = r.Store.AppendJobLog(ctx, job.ID, chunk)
131
}
132
return false, err
133
}
134
135
// RemoteInputs zet de artifacts van eerdere jobs in dir.
136
func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {
137
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
138
if err != nil {
139
return err
140
}
141
return r.materialize(ctx, p, job, dir)
142
}
143
144
// RemoteArtifacts bewaart de geüploade artifacts uit dir.
145
func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {
146
return r.collect(ctx, job, dir)
147
}
148
149
// RemoteFinish rondt een job van een aparte runner af.
150
func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {
151
job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)
152
if err != nil {
153
return err
154
}
155
if job.Status != "running" {
156
return nil
157
}
158
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)
159
if err != nil {
160
return err
161
}
162
var mask []string
163
if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {
164
mask = m.([]string)
165
}
166
extra, kind := "", ""
167
switch status {
168
case "success":
169
case "canceled":
170
extra = stamp("canceled")
171
default:
172
status = "failed"
173
if message == "" {
174
message = "job failed"
175
}
176
kind = classifyFailure(errors.New(message))
177
if strings.Contains(message, "job_execution_timeout") {
178
kind = "job_execution_timeout"
179
}
180
extra = stamp(ci.Mask(message, mask))
181
}
182
r.complete(p, job, p, status, extra, kind)
183
return nil
184
}
185
186
// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.
187
func (r *Runner) reapRemote(ctx context.Context) {
188
jobs, err := r.Store.RemoteRunningJobs(ctx)
189
if err != nil {
190
return
191
}
192
now := time.Now()
193
for _, rj := range jobs {
194
rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)
195
msg := ""
196
switch {
197
case err != nil || rn.Revoked:
198
msg = "runner_system_failure: runner was removed"
199
case now.Sub(rn.LastSeen) > RemoteOnlineGrace:
200
msg = "runner_system_failure: runner " + rn.Name + " stopped responding"
201
default:
202
limit := r.Timeout
203
if rj.Job.Timeout > 0 {
204
limit = rj.Job.Timeout
205
}
206
if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {
207
msg = "job_execution_timeout: no result from runner " + rn.Name
208
}
209
}
210
if msg != "" {
211
_ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)
212
}
213
}
214
}
215
216
// RunRemote draait een job op de runnermachine: clone, cache, inputs,
217
// container, cache opslaan. De werkmap blijft staan voor de artifacts.
218
// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.
219
func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {
220
sink := &logSink{out: out, secrets: pl.Mask}
221
sink.section("prepare", "Preparing the docker executor")
222
_ = os.RemoveAll(work)
223
if err := os.MkdirAll(work, 0o755); err != nil {
224
sink.closeSection()
225
return err
226
}
227
if err := clone(ctx, work); err != nil {
228
sink.closeSection()
229
return fmt.Errorf("clone: %w", err)
230
}
231
if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {
232
sink.closeSection()
233
return fmt.Errorf("checkout %s: %w", pl.SHA, err)
234
}
235
if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {
236
sink.closeSection()
237
return err
238
}
239
if err := inputs(ctx, work); err != nil {
240
sink.closeSection()
241
return fmt.Errorf("artifacts: %w", err)
242
}
243
if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {
244
sink.closeSection()
245
return err
246
}
247
if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {
248
return err
249
}
250
for _, rel := range pl.Job.ArtifactPaths {
251
if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {
252
return fmt.Errorf("artifact %s ontbreekt", rel)
253
}
254
}
255
return r.saveCache(pl.RepoID, pl.Job, work)
256
}
257
258
// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.
259
// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.
260
func WriteTar(w io.Writer, dir string, rels []string) error {
261
gz := gzip.NewWriter(w)
262
tw := tar.NewWriter(gz)
263
add := func(rel string) error {
264
path := filepath.Join(dir, filepath.FromSlash(rel))
265
info, err := os.Stat(path)
266
if err != nil || !info.Mode().IsRegular() {
267
return nil
268
}
269
f, err := os.Open(path)
270
if err != nil {
271
return err
272
}
273
defer f.Close()
274
if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
275
return err
276
}
277
_, err = io.Copy(tw, f)
278
return err
279
}
280
if rels == nil {
281
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
282
if err != nil || info.IsDir() {
283
return err
284
}
285
rel, _ := filepath.Rel(dir, path)
286
return add(filepath.ToSlash(rel))
287
})
288
if err != nil {
289
return err
290
}
291
} else {
292
for _, rel := range rels {
293
if err := add(rel); err != nil {
294
return err
295
}
296
}
297
}
298
if err := tw.Close(); err != nil {
299
return err
300
}
301
return gz.Close()
302
}
303
304
// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;
305
// met allowed niet-nil alleen die paden. max begrenst de totale grootte.
306
func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {
307
gz, err := gzip.NewReader(rd)
308
if err != nil {
309
return err
310
}
311
tr := tar.NewReader(gz)
312
ok := map[string]bool{}
313
for _, a := range allowed {
314
ok[filepath.ToSlash(filepath.Clean(a))] = true
315
}
316
var total int64
317
for {
318
h, err := tr.Next()
319
if errors.Is(err, io.EOF) {
320
return nil
321
}
322
if err != nil {
323
return err
324
}
325
if h.Typeflag != tar.TypeReg {
326
continue
327
}
328
name := filepath.ToSlash(filepath.Clean(h.Name))
329
if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {
330
return fmt.Errorf("unsafe path %q", h.Name)
331
}
332
if allowed != nil && !ok[name] {
333
continue
334
}
335
total += h.Size
336
if max > 0 && total > max {
337
return errors.New("archive too large")
338
}
339
dst := filepath.Join(dir, filepath.FromSlash(name))
340
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
341
return err
342
}
343
f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
344
if err != nil {
345
return err
346
}
347
if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {
348
f.Close()
349
return err
350
}
351
f.Close()
352
}
353
}
@@ -0,0 +1,353 @@1
package runner2
3
import (4
"archive/tar"5
"compress/gzip"6
"context"7
"errors"8
"fmt"9
"io"10
"os"11
"path/filepath"12
"strings"13
"sync"14
"time"15
16
"gitlab.com/Timfa/bluegit/internal/ci"17
"gitlab.com/Timfa/bluegit/internal/store"18
)19
20
// RemotePayload is wat een aparte runner bij een job krijgt.21
type RemotePayload struct {22
JobID int64 `json:"job_id"`23
PipelineID int64 `json:"pipeline_id"`24
RepoID int64 `json:"repo_id"`25
Job store.Job `json:"job"`26
CloneURL string `json:"clone_url"`27
SHA string `json:"sha"`28
Script string `json:"script"`29
Env map[string]string `json:"env"`30
Mask []string `json:"mask"`31
Timeout int64 `json:"timeout_seconds"`32
}33
34
var remoteMasks sync.Map // job id → []string35
36
// RemoteOnlineGrace: zo lang mag een runner stil zijn met een lopende job.37
var RemoteOnlineGrace = 5 * time.Minute38
39
// Claim zoekt een aangeboden job voor runner rn en zet hem op running.40
// nil, nil als er niets is.41
func (r *Runner) Claim(ctx context.Context, rn store.Runner) (*RemotePayload, error) {42
if r.storage() == "" {43
return nil, nil44
}45
jobs, runs, err := r.Store.OfferedJobs(ctx)46
if err != nil {47
return nil, err48
}49
cache := map[int64][]Tier{}50
for _, j := range jobs {51
tiers := r.tiersFor(ctx, cache, j.RepoID)52
if !RunnerInTier(tiers, runs[j.ID].OfferTier, rn.ID) {53
continue54
}55
repo, err := r.Store.RepoByID(ctx, j.RepoID)56
if err != nil || !r.mayRun(ctx, rn, repo) {57
continue58
}59
p, err := r.Store.PipelineByID(ctx, j.RepoID, j.PipelineID)60
if err != nil || (r.Hold != nil && r.Hold(p)) {61
continue62
}63
if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {64
continue65
}66
ok, err := r.Store.ClaimJob(ctx, j.ID, rn.ID, rn.Name)67
if err != nil || !ok {68
continue69
}70
pl, err := r.payload(ctx, repo, p, j, rn)71
if err != nil {72
r.RemoteFinish(context.Background(), rn, j.ID, "failed", "runner_system_failure: "+err.Error())73
continue74
}75
return pl, nil76
}77
return nil, nil78
}79
80
// mayRun is de privacycontrole: een account-runner krijgt alleen jobs81
// uit repo's die zijn eigenaar mag lezen; een groepsrunner alleen uit82
// repo's van die groep.83
func (r *Runner) mayRun(ctx context.Context, rn store.Runner, repo store.Repo) bool {84
switch rn.OwnerKind {85
case "group":86
return repo.OwnerKind == "group" && repo.OwnerID == rn.OwnerID87
case "user":88
if repo.OwnerKind == "user" {89
return repo.OwnerID == rn.OwnerID90
}91
role, err := r.Store.EffectiveRole(ctx, rn.OwnerID, repo.ID)92
return err == nil && role.CanRead()93
}94
return false95
}96
97
func (r *Runner) payload(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, rn store.Runner) (*RemotePayload, error) {98
token, err := r.Store.CreateJobToken(ctx, job.ID)99
if err != nil {100
return nil, err101
}102
env, mask, err := r.environment(ctx, repo, p, job, token)103
if err != nil {104
return nil, err105
}106
remoteMasks.Store(job.ID, mask)107
limit := r.Timeout108
if job.Timeout > 0 {109
limit = job.Timeout110
}111
script := shellScript(job, placeNote{Runner: rn.Name, Host: rn.Name, WorkDir: "/workspace", PipelineID: p.ID, JobID: job.ID})112
return &RemotePayload{113
JobID: job.ID, PipelineID: p.ID, RepoID: repo.ID, Job: job,114
CloneURL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + ".git",115
SHA: p.CommitSHA, Script: script, Env: env, Mask: mask,116
Timeout: int64(limit / time.Second),117
}, nil118
}119
120
// RemoteLog voegt logregels van een aparte runner toe. cancel is true als121
// de job niet meer loopt (geannuleerd of afgebroken).122
func (r *Runner) RemoteLog(ctx context.Context, job store.Job, chunk string) (cancel bool, err error) {123
if job.Status != "running" {124
return true, nil125
}126
if m, ok := remoteMasks.Load(job.ID); ok {127
chunk = ci.Mask(chunk, m.([]string))128
}129
if len(chunk) > 0 {130
err = r.Store.AppendJobLog(ctx, job.ID, chunk)131
}132
return false, err133
}134
135
// RemoteInputs zet de artifacts van eerdere jobs in dir.136
func (r *Runner) RemoteInputs(ctx context.Context, job store.Job, dir string) error {137
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)138
if err != nil {139
return err140
}141
return r.materialize(ctx, p, job, dir)142
}143
144
// RemoteArtifacts bewaart de geüploade artifacts uit dir.145
func (r *Runner) RemoteArtifacts(ctx context.Context, job store.Job, dir string) error {146
return r.collect(ctx, job, dir)147
}148
149
// RemoteFinish rondt een job van een aparte runner af.150
func (r *Runner) RemoteFinish(ctx context.Context, rn store.Runner, jobID int64, status, message string) error {151
job, err := r.Store.JobForRunner(ctx, rn.ID, jobID)152
if err != nil {153
return err154
}155
if job.Status != "running" {156
return nil157
}158
p, err := r.Store.PipelineByID(ctx, job.RepoID, job.PipelineID)159
if err != nil {160
return err161
}162
var mask []string163
if m, ok := remoteMasks.LoadAndDelete(job.ID); ok {164
mask = m.([]string)165
}166
extra, kind := "", ""167
switch status {168
case "success":169
case "canceled":170
extra = stamp("canceled")171
default:172
status = "failed"173
if message == "" {174
message = "job failed"175
}176
kind = classifyFailure(errors.New(message))177
if strings.Contains(message, "job_execution_timeout") {178
kind = "job_execution_timeout"179
}180
extra = stamp(ci.Mask(message, mask))181
}182
r.complete(p, job, p, status, extra, kind)183
return nil184
}185
186
// reapRemote rondt jobs af van runners die verdwenen of te lang bezig zijn.187
func (r *Runner) reapRemote(ctx context.Context) {188
jobs, err := r.Store.RemoteRunningJobs(ctx)189
if err != nil {190
return191
}192
now := time.Now()193
for _, rj := range jobs {194
rn, err := r.Store.RunnerByID(ctx, rj.RunnerID)195
msg := ""196
switch {197
case err != nil || rn.Revoked:198
msg = "runner_system_failure: runner was removed"199
case now.Sub(rn.LastSeen) > RemoteOnlineGrace:200
msg = "runner_system_failure: runner " + rn.Name + " stopped responding"201
default:202
limit := r.Timeout203
if rj.Job.Timeout > 0 {204
limit = rj.Job.Timeout205
}206
if limit > 0 && !rj.Job.StartedAt.IsZero() && now.Sub(rj.Job.StartedAt) > limit+RemoteOnlineGrace {207
msg = "job_execution_timeout: no result from runner " + rn.Name208
}209
}210
if msg != "" {211
_ = r.RemoteFinish(ctx, store.Runner{ID: rj.RunnerID}, rj.Job.ID, "failed", msg)212
}213
}214
}215
216
// RunRemote draait een job op de runnermachine: clone, cache, inputs,217
// container, cache opslaan. De werkmap blijft staan voor de artifacts.218
// clone haalt de broncode binnen; inputs zet artifacts van eerdere jobs.219
func (r *Runner) RunRemote(ctx context.Context, pl RemotePayload, work string, out func(string), clone func(ctx context.Context, work string) error, inputs func(ctx context.Context, dir string) error) error {220
sink := &logSink{out: out, secrets: pl.Mask}221
sink.section("prepare", "Preparing the docker executor")222
_ = os.RemoveAll(work)223
if err := os.MkdirAll(work, 0o755); err != nil {224
sink.closeSection()225
return err226
}227
if err := clone(ctx, work); err != nil {228
sink.closeSection()229
return fmt.Errorf("clone: %w", err)230
}231
if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", pl.SHA); err != nil {232
sink.closeSection()233
return fmt.Errorf("checkout %s: %w", pl.SHA, err)234
}235
if err := r.restoreCache(pl.RepoID, pl.Job, work); err != nil {236
sink.closeSection()237
return err238
}239
if err := inputs(ctx, work); err != nil {240
sink.closeSection()241
return fmt.Errorf("artifacts: %w", err)242
}243
if err := os.WriteFile(filepath.Join(work, ".bluegit-job.sh"), []byte(pl.Script), 0o755); err != nil {244
sink.closeSection()245
return err246
}247
if err := r.execContainer(ctx, sink, pl.Job, work, pl.Env); err != nil {248
return err249
}250
for _, rel := range pl.Job.ArtifactPaths {251
if info, err := os.Stat(filepath.Join(work, filepath.FromSlash(rel))); err != nil || info.IsDir() {252
return fmt.Errorf("artifact %s ontbreekt", rel)253
}254
}255
return r.saveCache(pl.RepoID, pl.Job, work)256
}257
258
// WriteTar schrijft de bestanden rels (relatief aan dir) als tar.gz.259
// Ontbrekende bestanden worden overgeslagen. Met rels nil gaat alles mee.260
func WriteTar(w io.Writer, dir string, rels []string) error {261
gz := gzip.NewWriter(w)262
tw := tar.NewWriter(gz)263
add := func(rel string) error {264
path := filepath.Join(dir, filepath.FromSlash(rel))265
info, err := os.Stat(path)266
if err != nil || !info.Mode().IsRegular() {267
return nil268
}269
f, err := os.Open(path)270
if err != nil {271
return err272
}273
defer f.Close()274
if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {275
return err276
}277
_, err = io.Copy(tw, f)278
return err279
}280
if rels == nil {281
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {282
if err != nil || info.IsDir() {283
return err284
}285
rel, _ := filepath.Rel(dir, path)286
return add(filepath.ToSlash(rel))287
})288
if err != nil {289
return err290
}291
} else {292
for _, rel := range rels {293
if err := add(rel); err != nil {294
return err295
}296
}297
}298
if err := tw.Close(); err != nil {299
return err300
}301
return gz.Close()302
}303
304
// ReadTar pakt een tar.gz uit in dir. Alleen gewone bestanden binnen dir;305
// met allowed niet-nil alleen die paden. max begrenst de totale grootte.306
func ReadTar(rd io.Reader, dir string, allowed []string, max int64) error {307
gz, err := gzip.NewReader(rd)308
if err != nil {309
return err310
}311
tr := tar.NewReader(gz)312
ok := map[string]bool{}313
for _, a := range allowed {314
ok[filepath.ToSlash(filepath.Clean(a))] = true315
}316
var total int64317
for {318
h, err := tr.Next()319
if errors.Is(err, io.EOF) {320
return nil321
}322
if err != nil {323
return err324
}325
if h.Typeflag != tar.TypeReg {326
continue327
}328
name := filepath.ToSlash(filepath.Clean(h.Name))329
if name == "." || strings.HasPrefix(name, "../") || name == ".." || filepath.IsAbs(name) || strings.HasPrefix(name, "/") {330
return fmt.Errorf("unsafe path %q", h.Name)331
}332
if allowed != nil && !ok[name] {333
continue334
}335
total += h.Size336
if max > 0 && total > max {337
return errors.New("archive too large")338
}339
dst := filepath.Join(dir, filepath.FromSlash(name))340
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {341
return err342
}343
f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)344
if err != nil {345
return err346
}347
if _, err := io.Copy(f, io.LimitReader(tr, h.Size)); err != nil {348
f.Close()349
return err350
}351
f.Close()352
}353
}@@ -0,0 +1,149 @@
1
package runner
2
3
import (
4
"context"
5
"time"
6
7
"gitlab.com/Timfa/bluegit/internal/access"
8
"gitlab.com/Timfa/bluegit/internal/store"
9
)
10
11
// Tier is één trede in de runnervolgorde van een job: de ingebouwde
12
// BlueGit-runner of een set aparte runners.
13
type Tier struct {
14
Builtin bool
15
Runners []store.Runner
16
}
17
18
// OfferTimeout: zo lang krijgen online aparte runners in een trede om een
19
// job op te pakken voordat de volgende trede aan de beurt is.
20
var OfferTimeout = 20 * time.Second
21
22
// Order berekent de volgorde puur uit de gegevens; zie Tiers.
23
//
24
// Accountrepo: eigen runners eerst (own_first) of de BlueGit-runner eerst
25
// en eigen runners alleen als die niet meteen vrij is (builtin_first).
26
// Groepsrepo: groepsrunners → account-runners van maintainers die
27
// toestemming gaven → BlueGit-runner; met builtinFirst staat de
28
// BlueGit-runner vooraan. Lege treden vallen weg.
29
func Order(ownerKind, userPref string, builtinFirst bool, own, group, optedIn []store.Runner) []Tier {
30
var tiers []Tier
31
add := func(t Tier) {
32
if t.Builtin || len(t.Runners) > 0 {
33
tiers = append(tiers, t)
34
}
35
}
36
builtin := Tier{Builtin: true}
37
if ownerKind == "group" {
38
if builtinFirst {
39
add(builtin)
40
}
41
add(Tier{Runners: group})
42
add(Tier{Runners: optedIn})
43
if !builtinFirst {
44
add(builtin)
45
}
46
return tiers
47
}
48
if userPref == store.RunnerPrefOwnFirst {
49
add(Tier{Runners: own})
50
add(builtin)
51
} else {
52
add(builtin)
53
add(Tier{Runners: own})
54
}
55
return tiers
56
}
57
58
// Tiers geeft de runnervolgorde voor een repo.
59
func Tiers(ctx context.Context, st *store.Store, repo store.Repo) ([]Tier, error) {
60
if repo.OwnerKind == "group" {
61
group, err := st.ListRunners(ctx, "group", repo.OwnerID)
62
if err != nil {
63
return nil, err
64
}
65
users, err := st.RunnerOptIns(ctx, repo.OwnerID)
66
if err != nil {
67
return nil, err
68
}
69
var opted []store.Runner
70
for _, u := range users {
71
// Toestemming geldt alleen zolang de gebruiker maintainer is.
72
role, err := st.GroupRole(ctx, repo.OwnerID, u)
73
if err != nil || !role.AtLeast(access.RoleMaintainer) {
74
continue
75
}
76
rs, err := st.ListRunners(ctx, "user", u)
77
if err != nil {
78
return nil, err
79
}
80
opted = append(opted, rs...)
81
}
82
return Order("group", "", st.GroupBuiltinFirst(ctx, repo.OwnerID), nil, group, opted), nil
83
}
84
own, err := st.ListRunners(ctx, "user", repo.OwnerID)
85
if err != nil {
86
return nil, err
87
}
88
return Order("user", st.UserRunnerPref(ctx, repo.OwnerID), false, own, nil, nil), nil
89
}
90
91
func anyOnline(rs []store.Runner, now time.Time) bool {
92
for _, r := range rs {
93
if r.Online(now) {
94
return true
95
}
96
}
97
return false
98
}
99
100
// Step is de beslissing voor een wachtende job in deze tick.
101
type Step struct {
102
Tier int // trede waar de job nu staat
103
Builtin bool // nu ingebouwd starten
104
Moved bool // aanbod is verplaatst; opslaan
105
}
106
107
// Advance kiest wat er met een wachtende job gebeurt. builtinFree zegt of
108
// de ingebouwde runner nu een plek heeft. Elke trede wordt hooguit één
109
// keer per tick bekeken.
110
func Advance(tiers []Tier, tier int, offerAt, now time.Time, builtinFree bool) Step {
111
n := len(tiers)
112
if n == 0 {
113
return Step{}
114
}
115
if tier < 0 || tier >= n {
116
tier = 0
117
}
118
moved := offerAt.IsZero()
119
if moved {
120
offerAt = now
121
}
122
for i := 0; i < n; i++ {
123
t := tiers[tier]
124
if t.Builtin {
125
if builtinFree {
126
return Step{Tier: tier, Builtin: true, Moved: moved}
127
}
128
} else if anyOnline(t.Runners, now) && (now.Sub(offerAt) < OfferTimeout || i > 0 && moved) {
129
return Step{Tier: tier, Moved: moved}
130
}
131
tier = (tier + 1) % n
132
offerAt = now
133
moved = true
134
}
135
return Step{Tier: tier, Moved: moved}
136
}
137
138
// RunnerInTier zegt of runner id in de trede staat.
139
func RunnerInTier(tiers []Tier, tier int, id int64) bool {
140
if tier < 0 || tier >= len(tiers) {
141
return false
142
}
143
for _, r := range tiers[tier].Runners {
144
if r.ID == id {
145
return true
146
}
147
}
148
return false
149
}
@@ -0,0 +1,149 @@1
package runner2
3
import (4
"context"5
"time"6
7
"gitlab.com/Timfa/bluegit/internal/access"8
"gitlab.com/Timfa/bluegit/internal/store"9
)10
11
// Tier is één trede in de runnervolgorde van een job: de ingebouwde12
// BlueGit-runner of een set aparte runners.13
type Tier struct {14
Builtin bool15
Runners []store.Runner16
}17
18
// OfferTimeout: zo lang krijgen online aparte runners in een trede om een19
// job op te pakken voordat de volgende trede aan de beurt is.20
var OfferTimeout = 20 * time.Second21
22
// Order berekent de volgorde puur uit de gegevens; zie Tiers.23
//24
// Accountrepo: eigen runners eerst (own_first) of de BlueGit-runner eerst25
// en eigen runners alleen als die niet meteen vrij is (builtin_first).26
// Groepsrepo: groepsrunners → account-runners van maintainers die27
// toestemming gaven → BlueGit-runner; met builtinFirst staat de28
// BlueGit-runner vooraan. Lege treden vallen weg.29
func Order(ownerKind, userPref string, builtinFirst bool, own, group, optedIn []store.Runner) []Tier {30
var tiers []Tier31
add := func(t Tier) {32
if t.Builtin || len(t.Runners) > 0 {33
tiers = append(tiers, t)34
}35
}36
builtin := Tier{Builtin: true}37
if ownerKind == "group" {38
if builtinFirst {39
add(builtin)40
}41
add(Tier{Runners: group})42
add(Tier{Runners: optedIn})43
if !builtinFirst {44
add(builtin)45
}46
return tiers47
}48
if userPref == store.RunnerPrefOwnFirst {49
add(Tier{Runners: own})50
add(builtin)51
} else {52
add(builtin)53
add(Tier{Runners: own})54
}55
return tiers56
}57
58
// Tiers geeft de runnervolgorde voor een repo.59
func Tiers(ctx context.Context, st *store.Store, repo store.Repo) ([]Tier, error) {60
if repo.OwnerKind == "group" {61
group, err := st.ListRunners(ctx, "group", repo.OwnerID)62
if err != nil {63
return nil, err64
}65
users, err := st.RunnerOptIns(ctx, repo.OwnerID)66
if err != nil {67
return nil, err68
}69
var opted []store.Runner70
for _, u := range users {71
// Toestemming geldt alleen zolang de gebruiker maintainer is.72
role, err := st.GroupRole(ctx, repo.OwnerID, u)73
if err != nil || !role.AtLeast(access.RoleMaintainer) {74
continue75
}76
rs, err := st.ListRunners(ctx, "user", u)77
if err != nil {78
return nil, err79
}80
opted = append(opted, rs...)81
}82
return Order("group", "", st.GroupBuiltinFirst(ctx, repo.OwnerID), nil, group, opted), nil83
}84
own, err := st.ListRunners(ctx, "user", repo.OwnerID)85
if err != nil {86
return nil, err87
}88
return Order("user", st.UserRunnerPref(ctx, repo.OwnerID), false, own, nil, nil), nil89
}90
91
func anyOnline(rs []store.Runner, now time.Time) bool {92
for _, r := range rs {93
if r.Online(now) {94
return true95
}96
}97
return false98
}99
100
// Step is de beslissing voor een wachtende job in deze tick.101
type Step struct {102
Tier int // trede waar de job nu staat103
Builtin bool // nu ingebouwd starten104
Moved bool // aanbod is verplaatst; opslaan105
}106
107
// Advance kiest wat er met een wachtende job gebeurt. builtinFree zegt of108
// de ingebouwde runner nu een plek heeft. Elke trede wordt hooguit één109
// keer per tick bekeken.110
func Advance(tiers []Tier, tier int, offerAt, now time.Time, builtinFree bool) Step {111
n := len(tiers)112
if n == 0 {113
return Step{}114
}115
if tier < 0 || tier >= n {116
tier = 0117
}118
moved := offerAt.IsZero()119
if moved {120
offerAt = now121
}122
for i := 0; i < n; i++ {123
t := tiers[tier]124
if t.Builtin {125
if builtinFree {126
return Step{Tier: tier, Builtin: true, Moved: moved}127
}128
} else if anyOnline(t.Runners, now) && (now.Sub(offerAt) < OfferTimeout || i > 0 && moved) {129
return Step{Tier: tier, Moved: moved}130
}131
tier = (tier + 1) % n132
offerAt = now133
moved = true134
}135
return Step{Tier: tier, Moved: moved}136
}137
138
// RunnerInTier zegt of runner id in de trede staat.139
func RunnerInTier(tiers []Tier, tier int, id int64) bool {140
if tier < 0 || tier >= len(tiers) {141
return false142
}143
for _, r := range tiers[tier].Runners {144
if r.ID == id {145
return true146
}147
}148
return false149
}@@ -0,0 +1,191 @@
1
package runner
2
3
import (
4
"context"
5
"path/filepath"
6
"testing"
7
"time"
8
9
"gitlab.com/Timfa/bluegit/internal/access"
10
"gitlab.com/Timfa/bluegit/internal/store"
11
)
12
13
func names(tiers []Tier) []string {
14
var out []string
15
for _, t := range tiers {
16
if t.Builtin {
17
out = append(out, "builtin")
18
continue
19
}
20
s := ""
21
for i, r := range t.Runners {
22
if i > 0 {
23
s += "+"
24
}
25
s += r.Name
26
}
27
out = append(out, s)
28
}
29
return out
30
}
31
32
func eq(t *testing.T, got []string, want ...string) {
33
t.Helper()
34
if len(got) != len(want) {
35
t.Fatalf("got %v, want %v", got, want)
36
}
37
for i := range got {
38
if got[i] != want[i] {
39
t.Fatalf("got %v, want %v", got, want)
40
}
41
}
42
}
43
44
func TestOrder(t *testing.T) {
45
own := []store.Runner{{ID: 1, Name: "laptop"}}
46
grp := []store.Runner{{ID: 2, Name: "grp"}}
47
opt := []store.Runner{{ID: 3, Name: "maint"}}
48
eq(t, names(Order("user", store.RunnerPrefOwnFirst, false, own, nil, nil)), "laptop", "builtin")
49
eq(t, names(Order("user", store.RunnerPrefBuiltinFirst, false, own, nil, nil)), "builtin", "laptop")
50
eq(t, names(Order("user", "", false, own, nil, nil)), "builtin", "laptop")
51
eq(t, names(Order("user", store.RunnerPrefOwnFirst, false, nil, nil, nil)), "builtin")
52
eq(t, names(Order("group", "", false, nil, grp, opt)), "grp", "maint", "builtin")
53
eq(t, names(Order("group", "", true, nil, grp, opt)), "builtin", "grp", "maint")
54
eq(t, names(Order("group", "", false, nil, nil, opt)), "maint", "builtin")
55
eq(t, names(Order("group", "", false, nil, grp, nil)), "grp", "builtin")
56
}
57
58
func TestAdvance(t *testing.T) {
59
now := time.Now()
60
on := store.Runner{ID: 1, Name: "on", LastSeen: now}
61
off := store.Runner{ID: 2, Name: "off", LastSeen: now.Add(-time.Hour)}
62
remoteOn := Tier{Runners: []store.Runner{on}}
63
remoteOff := Tier{Runners: []store.Runner{off}}
64
builtin := Tier{Builtin: true}
65
66
// Eigen runners eerst, runner online: blijft op de runner wachten.
67
st := Advance([]Tier{remoteOn, builtin}, 0, time.Time{}, now, true)
68
if st.Builtin || st.Tier != 0 || !st.Moved {
69
t.Fatalf("own first online: %+v", st)
70
}
71
// Na de wachttijd gaat de job naar de BlueGit-runner.
72
st = Advance([]Tier{remoteOn, builtin}, 0, now.Add(-OfferTimeout-time.Second), now, true)
73
if !st.Builtin || st.Tier != 1 {
74
t.Fatalf("own first timeout: %+v", st)
75
}
76
// Runner offline: meteen door.
77
st = Advance([]Tier{remoteOff, builtin}, 0, time.Time{}, now, true)
78
if !st.Builtin || st.Tier != 1 {
79
t.Fatalf("offline: %+v", st)
80
}
81
// BlueGit-runner eerst maar vol: door naar de eigen runner.
82
st = Advance([]Tier{builtin, remoteOn}, 0, time.Time{}, now, false)
83
if st.Builtin || st.Tier != 1 || !st.Moved {
84
t.Fatalf("builtin busy: %+v", st)
85
}
86
// BlueGit-runner eerst en vrij: direct.
87
st = Advance([]Tier{builtin, remoteOn}, 0, time.Time{}, now, true)
88
if !st.Builtin || st.Tier != 0 {
89
t.Fatalf("builtin free: %+v", st)
90
}
91
// Groep: groepsrunner offline, maintainer-runner online → wacht daar.
92
st = Advance([]Tier{remoteOff, remoteOn, builtin}, 0, time.Time{}, now, true)
93
if st.Builtin || st.Tier != 1 {
94
t.Fatalf("group fallback: %+v", st)
95
}
96
// Alles bezet en offline: rondje, eindigt weer bij een trede, start niets.
97
st = Advance([]Tier{builtin, remoteOff}, 0, time.Time{}, now, false)
98
if st.Builtin {
99
t.Fatalf("nothing free: %+v", st)
100
}
101
// Wrap: maintainer-runner timeout, builtin vol → terug naar de groepsrunner (online).
102
st = Advance([]Tier{remoteOn, builtin}, 1, now, now, false)
103
if st.Builtin || st.Tier != 0 {
104
t.Fatalf("wrap: %+v", st)
105
}
106
}
107
108
func TestTiersAndPrivacy(t *testing.T) {
109
dir := t.TempDir()
110
s, err := store.Open(filepath.Join(dir, "b.db"), make([]byte, 32))
111
if err != nil {
112
t.Fatal(err)
113
}
114
defer s.Close()
115
ctx := context.Background()
116
tim, _ := s.UpsertUser(ctx, 1, "tim", "", false)
117
bob, _ := s.UpsertUser(ctx, 2, "bob", "", false)
118
carol, _ := s.UpsertUser(ctx, 3, "carol", "", false)
119
g, err := s.CreateGroup(ctx, tim.ID, "Team", "team", "", access.RoleReadonly)
120
if err != nil {
121
t.Fatal(err)
122
}
123
if err := s.SetMember(ctx, "group", g.ID, "bob", access.RoleDeveloper); err != nil {
124
t.Fatal(err)
125
}
126
if err := s.SetMember(ctx, "group", g.ID, "carol", access.RoleMaintainer); err != nil {
127
t.Fatal(err)
128
}
129
grepo, err := s.CreateRepo(ctx, "group", g.ID, "app", "", "master", "private")
130
if err != nil {
131
t.Fatal(err)
132
}
133
trepo, err := s.CreateRepo(ctx, "user", tim.ID, "secret", "", "master", "private")
134
if err != nil {
135
t.Fatal(err)
136
}
137
reg := func(kind string, id int64, name string) store.Runner {
138
tok, err := s.CreateRunnerRegToken(ctx, kind, id, id)
139
if err != nil {
140
t.Fatal(err)
141
}
142
rn, _, err := s.RegisterRunner(ctx, tok, name, "test")
143
if err != nil {
144
t.Fatal(err)
145
}
146
if _, _, err := s.RegisterRunner(ctx, tok, "again", "test"); err == nil {
147
t.Fatal("registration token reused")
148
}
149
return rn
150
}
151
grn := reg("group", g.ID, "grp")
152
bobRn := reg("user", bob.ID, "bob-pc")
153
carolRn := reg("user", carol.ID, "carol-pc")
154
timRn := reg("user", tim.ID, "tim-pc")
155
_ = s.SetRunnerOptIn(ctx, g.ID, bob.ID, true) // geen maintainer: telt niet
156
_ = s.SetRunnerOptIn(ctx, g.ID, carol.ID, true) // maintainer: telt
157
tiers, err := Tiers(ctx, s, grepo)
158
if err != nil {
159
t.Fatal(err)
160
}
161
eq(t, names(tiers), "grp", "carol-pc", "builtin")
162
_ = s.SetGroupBuiltinFirst(ctx, g.ID, true)
163
tiers, _ = Tiers(ctx, s, grepo)
164
eq(t, names(tiers), "builtin", "grp", "carol-pc")
165
166
tiers, _ = Tiers(ctx, s, trepo)
167
eq(t, names(tiers), "builtin", "tim-pc")
168
_ = s.SetUserRunnerPref(ctx, tim.ID, store.RunnerPrefOwnFirst)
169
tiers, _ = Tiers(ctx, s, trepo)
170
eq(t, names(tiers), "tim-pc", "builtin")
171
172
r := &Runner{Store: s}
173
if !r.mayRun(ctx, grn, grepo) || r.mayRun(ctx, grn, trepo) {
174
t.Fatal("group runner scope")
175
}
176
if !r.mayRun(ctx, carolRn, grepo) || r.mayRun(ctx, carolRn, trepo) {
177
t.Fatal("carol scope")
178
}
179
if r.mayRun(ctx, bobRn, trepo) {
180
t.Fatal("bob may not run tim's private repo")
181
}
182
if !r.mayRun(ctx, timRn, trepo) {
183
t.Fatal("tim runs own repo")
184
}
185
// Ingetrokken runner verdwijnt uit de volgorde en zijn token werkt niet meer.
186
if err := s.RevokeRunner(ctx, "user", tim.ID, timRn.ID); err != nil {
187
t.Fatal(err)
188
}
189
tiers, _ = Tiers(ctx, s, trepo)
190
eq(t, names(tiers), "builtin")
191
}
@@ -0,0 +1,191 @@1
package runner2
3
import (4
"context"5
"path/filepath"6
"testing"7
"time"8
9
"gitlab.com/Timfa/bluegit/internal/access"10
"gitlab.com/Timfa/bluegit/internal/store"11
)12
13
func names(tiers []Tier) []string {14
var out []string15
for _, t := range tiers {16
if t.Builtin {17
out = append(out, "builtin")18
continue19
}20
s := ""21
for i, r := range t.Runners {22
if i > 0 {23
s += "+"24
}25
s += r.Name26
}27
out = append(out, s)28
}29
return out30
}31
32
func eq(t *testing.T, got []string, want ...string) {33
t.Helper()34
if len(got) != len(want) {35
t.Fatalf("got %v, want %v", got, want)36
}37
for i := range got {38
if got[i] != want[i] {39
t.Fatalf("got %v, want %v", got, want)40
}41
}42
}43
44
func TestOrder(t *testing.T) {45
own := []store.Runner{{ID: 1, Name: "laptop"}}46
grp := []store.Runner{{ID: 2, Name: "grp"}}47
opt := []store.Runner{{ID: 3, Name: "maint"}}48
eq(t, names(Order("user", store.RunnerPrefOwnFirst, false, own, nil, nil)), "laptop", "builtin")49
eq(t, names(Order("user", store.RunnerPrefBuiltinFirst, false, own, nil, nil)), "builtin", "laptop")50
eq(t, names(Order("user", "", false, own, nil, nil)), "builtin", "laptop")51
eq(t, names(Order("user", store.RunnerPrefOwnFirst, false, nil, nil, nil)), "builtin")52
eq(t, names(Order("group", "", false, nil, grp, opt)), "grp", "maint", "builtin")53
eq(t, names(Order("group", "", true, nil, grp, opt)), "builtin", "grp", "maint")54
eq(t, names(Order("group", "", false, nil, nil, opt)), "maint", "builtin")55
eq(t, names(Order("group", "", false, nil, grp, nil)), "grp", "builtin")56
}57
58
func TestAdvance(t *testing.T) {59
now := time.Now()60
on := store.Runner{ID: 1, Name: "on", LastSeen: now}61
off := store.Runner{ID: 2, Name: "off", LastSeen: now.Add(-time.Hour)}62
remoteOn := Tier{Runners: []store.Runner{on}}63
remoteOff := Tier{Runners: []store.Runner{off}}64
builtin := Tier{Builtin: true}65
66
// Eigen runners eerst, runner online: blijft op de runner wachten.67
st := Advance([]Tier{remoteOn, builtin}, 0, time.Time{}, now, true)68
if st.Builtin || st.Tier != 0 || !st.Moved {69
t.Fatalf("own first online: %+v", st)70
}71
// Na de wachttijd gaat de job naar de BlueGit-runner.72
st = Advance([]Tier{remoteOn, builtin}, 0, now.Add(-OfferTimeout-time.Second), now, true)73
if !st.Builtin || st.Tier != 1 {74
t.Fatalf("own first timeout: %+v", st)75
}76
// Runner offline: meteen door.77
st = Advance([]Tier{remoteOff, builtin}, 0, time.Time{}, now, true)78
if !st.Builtin || st.Tier != 1 {79
t.Fatalf("offline: %+v", st)80
}81
// BlueGit-runner eerst maar vol: door naar de eigen runner.82
st = Advance([]Tier{builtin, remoteOn}, 0, time.Time{}, now, false)83
if st.Builtin || st.Tier != 1 || !st.Moved {84
t.Fatalf("builtin busy: %+v", st)85
}86
// BlueGit-runner eerst en vrij: direct.87
st = Advance([]Tier{builtin, remoteOn}, 0, time.Time{}, now, true)88
if !st.Builtin || st.Tier != 0 {89
t.Fatalf("builtin free: %+v", st)90
}91
// Groep: groepsrunner offline, maintainer-runner online → wacht daar.92
st = Advance([]Tier{remoteOff, remoteOn, builtin}, 0, time.Time{}, now, true)93
if st.Builtin || st.Tier != 1 {94
t.Fatalf("group fallback: %+v", st)95
}96
// Alles bezet en offline: rondje, eindigt weer bij een trede, start niets.97
st = Advance([]Tier{builtin, remoteOff}, 0, time.Time{}, now, false)98
if st.Builtin {99
t.Fatalf("nothing free: %+v", st)100
}101
// Wrap: maintainer-runner timeout, builtin vol → terug naar de groepsrunner (online).102
st = Advance([]Tier{remoteOn, builtin}, 1, now, now, false)103
if st.Builtin || st.Tier != 0 {104
t.Fatalf("wrap: %+v", st)105
}106
}107
108
func TestTiersAndPrivacy(t *testing.T) {109
dir := t.TempDir()110
s, err := store.Open(filepath.Join(dir, "b.db"), make([]byte, 32))111
if err != nil {112
t.Fatal(err)113
}114
defer s.Close()115
ctx := context.Background()116
tim, _ := s.UpsertUser(ctx, 1, "tim", "", false)117
bob, _ := s.UpsertUser(ctx, 2, "bob", "", false)118
carol, _ := s.UpsertUser(ctx, 3, "carol", "", false)119
g, err := s.CreateGroup(ctx, tim.ID, "Team", "team", "", access.RoleReadonly)120
if err != nil {121
t.Fatal(err)122
}123
if err := s.SetMember(ctx, "group", g.ID, "bob", access.RoleDeveloper); err != nil {124
t.Fatal(err)125
}126
if err := s.SetMember(ctx, "group", g.ID, "carol", access.RoleMaintainer); err != nil {127
t.Fatal(err)128
}129
grepo, err := s.CreateRepo(ctx, "group", g.ID, "app", "", "master", "private")130
if err != nil {131
t.Fatal(err)132
}133
trepo, err := s.CreateRepo(ctx, "user", tim.ID, "secret", "", "master", "private")134
if err != nil {135
t.Fatal(err)136
}137
reg := func(kind string, id int64, name string) store.Runner {138
tok, err := s.CreateRunnerRegToken(ctx, kind, id, id)139
if err != nil {140
t.Fatal(err)141
}142
rn, _, err := s.RegisterRunner(ctx, tok, name, "test")143
if err != nil {144
t.Fatal(err)145
}146
if _, _, err := s.RegisterRunner(ctx, tok, "again", "test"); err == nil {147
t.Fatal("registration token reused")148
}149
return rn150
}151
grn := reg("group", g.ID, "grp")152
bobRn := reg("user", bob.ID, "bob-pc")153
carolRn := reg("user", carol.ID, "carol-pc")154
timRn := reg("user", tim.ID, "tim-pc")155
_ = s.SetRunnerOptIn(ctx, g.ID, bob.ID, true) // geen maintainer: telt niet156
_ = s.SetRunnerOptIn(ctx, g.ID, carol.ID, true) // maintainer: telt157
tiers, err := Tiers(ctx, s, grepo)158
if err != nil {159
t.Fatal(err)160
}161
eq(t, names(tiers), "grp", "carol-pc", "builtin")162
_ = s.SetGroupBuiltinFirst(ctx, g.ID, true)163
tiers, _ = Tiers(ctx, s, grepo)164
eq(t, names(tiers), "builtin", "grp", "carol-pc")165
166
tiers, _ = Tiers(ctx, s, trepo)167
eq(t, names(tiers), "builtin", "tim-pc")168
_ = s.SetUserRunnerPref(ctx, tim.ID, store.RunnerPrefOwnFirst)169
tiers, _ = Tiers(ctx, s, trepo)170
eq(t, names(tiers), "tim-pc", "builtin")171
172
r := &Runner{Store: s}173
if !r.mayRun(ctx, grn, grepo) || r.mayRun(ctx, grn, trepo) {174
t.Fatal("group runner scope")175
}176
if !r.mayRun(ctx, carolRn, grepo) || r.mayRun(ctx, carolRn, trepo) {177
t.Fatal("carol scope")178
}179
if r.mayRun(ctx, bobRn, trepo) {180
t.Fatal("bob may not run tim's private repo")181
}182
if !r.mayRun(ctx, timRn, trepo) {183
t.Fatal("tim runs own repo")184
}185
// Ingetrokken runner verdwijnt uit de volgorde en zijn token werkt niet meer.186
if err := s.RevokeRunner(ctx, "user", tim.ID, timRn.ID); err != nil {187
t.Fatal(err)188
}189
tiers, _ = Tiers(ctx, s, trepo)190
eq(t, names(tiers), "builtin")191
}@@ -117,6 +117,7 @@ func (r *Runner) Tick(ctx context.Context) {
117
117
for _, p := range pipes {
118
118
r.schedule(ctx, p)
119
119
}
120
r.reapRemote(ctx)
120
121
}
121
122
122
123
func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
@@ -135,6 +136,7 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
135
136
specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
136
137
status[j.Name] = j.Status
137
138
}
139
tierCache := map[int64][]Tier{}
138
140
for _, j := range jobs {
139
141
if j.Status != "queued" && j.Status != "manual" {
140
142
continue
@@ -155,7 +157,21 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
155
157
}
156
158
// Meer jobs dan ci.concurrency blijven queued, met een plek
157
159
// in de wachtrij. Het slot gaat pas vrij als execute klaar is.
158
if !r.acquireSlot() {
160
free := r.acquireSlot()
161
if tiers := r.tiersFor(ctx, tierCache, j.RepoID); len(tiers) > 1 {
162
jr := r.Store.JobRun(ctx, j.ID)
163
step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
164
if !step.Builtin {
165
if free {
166
r.releaseSlot()
167
}
168
if step.Moved {
169
_ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
170
}
171
continue
172
}
173
}
174
if !free {
159
175
continue
160
176
}
161
177
ok, err := r.Store.MarkJobRunning(ctx, j.ID)
@@ -163,6 +179,8 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
163
179
r.releaseSlot()
164
180
continue
165
181
}
182
_ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
183
_ = r.Store.ClearJobOffer(ctx, j.ID)
166
184
status[j.Name] = "running"
167
185
job := j
168
186
pipe := p
@@ -217,6 +235,26 @@ func (r *Runner) settle(ctx context.Context, pipelineID int64) {
217
235
}
218
236
}
219
237
238
// BuiltinName is de naam van de ingebouwde runner in de job-UI.
239
func (r *Runner) BuiltinName() string {
240
if r.Name != "" {
241
return r.Name
242
}
243
return "BlueGit runner"
244
}
245
246
func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64) []Tier {
247
if t, ok := cache[repoID]; ok {
248
return t
249
}
250
var tiers []Tier
251
if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
252
tiers, _ = Tiers(ctx, r.Store, repo)
253
}
254
cache[repoID] = tiers
255
return tiers
256
}
257
220
258
func indexOf(jobs []store.Job, name string) int {
221
259
for i, j := range jobs {
222
260
if j.Name == name {
@@ -285,8 +323,15 @@ func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
285
323
kind = classifyFailure(err)
286
324
extra = stamp(ci.Mask(err.Error(), mask))
287
325
}
326
r.complete(p, job, before, status, extra, kind)
327
}
328
329
// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
330
// en webhook. Gedeeld door de ingebouwde en de aparte runner.
331
func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
288
332
if status == "failed" && shouldRetry(job, kind) {
289
333
note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
334
_ = r.Store.ClearJobOffer(context.Background(), job.ID)
290
335
if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
291
336
r.Log.Error("job opnieuw", "err", ferr)
292
337
} else if ferr == nil {
@@ -384,15 +429,31 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
384
429
return nil, err
385
430
}
386
431
sink.secrets = mask
432
if err := r.execContainer(ctx, sink, job, work, env); err != nil {
433
return mask, err
434
}
435
if err := r.collect(ctx, job, work); err != nil {
436
return mask, err
437
}
438
if err := r.saveCache(repo.ID, job, work); err != nil {
439
return mask, err
440
}
441
return mask, nil
442
}
443
444
// execContainer haalt het image op, start services en draait het
445
// jobscript in de container. De sectie prepare moet open staan.
446
func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
387
447
ref, lines, rerr := r.resolveImage(ctx, job.Image)
388
448
for _, line := range lines {
389
449
sink.emit(line)
390
450
}
391
451
if rerr != nil {
392
452
sink.closeSection()
393
return mask, rerr
453
return rerr
394
454
}
395
455
job.Image = ref
456
job.Services = append([]store.Service(nil), job.Services...)
396
457
for i := range job.Services {
397
458
sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
398
459
for _, line := range slines {
@@ -400,16 +461,19 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
400
461
}
401
462
if serr != nil {
402
463
sink.closeSection()
403
return mask, serr
464
return serr
404
465
}
405
466
job.Services[i].Name = sref
406
467
}
407
468
network := r.Network
469
if network == "" {
470
network = "bridge"
471
}
408
472
if len(job.Services) > 0 {
409
473
net, cleanup, serr := r.startServices(ctx, job)
410
474
if serr != nil {
411
475
sink.closeSection()
412
return mask, serr
476
return serr
413
477
}
414
478
defer cleanup()
415
479
network = net
@@ -429,18 +493,12 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([
429
493
cmd.Env = cmdEnv
430
494
cmd.Stdout = sink
431
495
cmd.Stderr = sink
432
err = cmd.Run()
496
err := cmd.Run()
433
497
sink.flush()
434
498
if err != nil {
435
return mask, fmt.Errorf("container ended with an error: %w", err)
499
return fmt.Errorf("container ended with an error: %w", err)
436
500
}
437
if err := r.collect(ctx, job, work); err != nil {
438
return mask, err
439
}
440
if err := r.saveCache(repo.ID, job, work); err != nil {
441
return mask, err
442
}
443
return mask, nil
501
return nil
444
502
}
445
503
446
504
// placeNote zijn de vaste regels in de sectie Preparing environment.
@@ -985,6 +1043,7 @@ type logSink struct {
985
1043
mu sync.Mutex
986
1044
store *store.Store
987
1045
jobID int64
1046
out func(string)
988
1047
secrets []string
989
1048
partial string
990
1049
current string
@@ -1028,6 +1087,10 @@ func (l *logSink) emit(line string) {
1028
1087
l.opened = time.Now()
1029
1088
}
1030
1089
}
1090
if l.out != nil {
1091
l.out(stamp(line))
1092
return
1093
}
1031
1094
_ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1032
1095
}
1033
1096
@@ -117,6 +117,7 @@ func (r *Runner) Tick(ctx context.Context) {117
for _, p := range pipes {117
for _, p := range pipes {118
r.schedule(ctx, p)118
r.schedule(ctx, p)119
}119
}120
r.reapRemote(ctx)120
}121
}121
122
122
func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {123
func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {@@ -135,6 +136,7 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {135
specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}136
specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}136
status[j.Name] = j.Status137
status[j.Name] = j.Status137
}138
}139
tierCache := map[int64][]Tier{}138
for _, j := range jobs {140
for _, j := range jobs {139
if j.Status != "queued" && j.Status != "manual" {141
if j.Status != "queued" && j.Status != "manual" {140
continue142
continue@@ -155,7 +157,21 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {155
}157
}156
// Meer jobs dan ci.concurrency blijven queued, met een plek158
// Meer jobs dan ci.concurrency blijven queued, met een plek157
// in de wachtrij. Het slot gaat pas vrij als execute klaar is.159
// in de wachtrij. Het slot gaat pas vrij als execute klaar is.158
if !r.acquireSlot() {160
free := r.acquireSlot()161
if tiers := r.tiersFor(ctx, tierCache, j.RepoID); len(tiers) > 1 {162
jr := r.Store.JobRun(ctx, j.ID)163
step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)164
if !step.Builtin {165
if free {166
r.releaseSlot()167
}168
if step.Moved {169
_ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())170
}171
continue172
}173
}174
if !free {159
continue175
continue160
}176
}161
ok, err := r.Store.MarkJobRunning(ctx, j.ID)177
ok, err := r.Store.MarkJobRunning(ctx, j.ID)@@ -163,6 +179,8 @@ func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {163
r.releaseSlot()179
r.releaseSlot()164
continue180
continue165
}181
}182
_ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())183
_ = r.Store.ClearJobOffer(ctx, j.ID)166
status[j.Name] = "running"184
status[j.Name] = "running"167
job := j185
job := j168
pipe := p186
pipe := p@@ -217,6 +235,26 @@ func (r *Runner) settle(ctx context.Context, pipelineID int64) {217
}235
}218
}236
}219
237
238
// BuiltinName is de naam van de ingebouwde runner in de job-UI.239
func (r *Runner) BuiltinName() string {240
if r.Name != "" {241
return r.Name242
}243
return "BlueGit runner"244
}245
246
func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64) []Tier {247
if t, ok := cache[repoID]; ok {248
return t249
}250
var tiers []Tier251
if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {252
tiers, _ = Tiers(ctx, r.Store, repo)253
}254
cache[repoID] = tiers255
return tiers256
}257
220
func indexOf(jobs []store.Job, name string) int {258
func indexOf(jobs []store.Job, name string) int {221
for i, j := range jobs {259
for i, j := range jobs {222
if j.Name == name {260
if j.Name == name {@@ -285,8 +323,15 @@ func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {285
kind = classifyFailure(err)323
kind = classifyFailure(err)286
extra = stamp(ci.Mask(err.Error(), mask))324
extra = stamp(ci.Mask(err.Error(), mask))287
}325
}326
r.complete(p, job, before, status, extra, kind)327
}328
329
// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus330
// en webhook. Gedeeld door de ingebouwde en de aparte runner.331
func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {288
if status == "failed" && shouldRetry(job, kind) {332
if status == "failed" && shouldRetry(job, kind) {289
note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))333
note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))334
_ = r.Store.ClearJobOffer(context.Background(), job.ID)290
if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {335
if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {291
r.Log.Error("job opnieuw", "err", ferr)336
r.Log.Error("job opnieuw", "err", ferr)292
} else if ferr == nil {337
} else if ferr == nil {@@ -384,15 +429,31 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([384
return nil, err429
return nil, err385
}430
}386
sink.secrets = mask431
sink.secrets = mask432
if err := r.execContainer(ctx, sink, job, work, env); err != nil {433
return mask, err434
}435
if err := r.collect(ctx, job, work); err != nil {436
return mask, err437
}438
if err := r.saveCache(repo.ID, job, work); err != nil {439
return mask, err440
}441
return mask, nil442
}443
444
// execContainer haalt het image op, start services en draait het445
// jobscript in de container. De sectie prepare moet open staan.446
func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {387
ref, lines, rerr := r.resolveImage(ctx, job.Image)447
ref, lines, rerr := r.resolveImage(ctx, job.Image)388
for _, line := range lines {448
for _, line := range lines {389
sink.emit(line)449
sink.emit(line)390
}450
}391
if rerr != nil {451
if rerr != nil {392
sink.closeSection()452
sink.closeSection()393
return mask, rerr453
return rerr394
}454
}395
job.Image = ref455
job.Image = ref456
job.Services = append([]store.Service(nil), job.Services...)396
for i := range job.Services {457
for i := range job.Services {397
sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)458
sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)398
for _, line := range slines {459
for _, line := range slines {@@ -400,16 +461,19 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([400
}461
}401
if serr != nil {462
if serr != nil {402
sink.closeSection()463
sink.closeSection()403
return mask, serr464
return serr404
}465
}405
job.Services[i].Name = sref466
job.Services[i].Name = sref406
}467
}407
network := r.Network468
network := r.Network469
if network == "" {470
network = "bridge"471
}408
if len(job.Services) > 0 {472
if len(job.Services) > 0 {409
net, cleanup, serr := r.startServices(ctx, job)473
net, cleanup, serr := r.startServices(ctx, job)410
if serr != nil {474
if serr != nil {411
sink.closeSection()475
sink.closeSection()412
return mask, serr476
return serr413
}477
}414
defer cleanup()478
defer cleanup()415
network = net479
network = net@@ -429,18 +493,12 @@ func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([429
cmd.Env = cmdEnv493
cmd.Env = cmdEnv430
cmd.Stdout = sink494
cmd.Stdout = sink431
cmd.Stderr = sink495
cmd.Stderr = sink432
err = cmd.Run()496
err := cmd.Run()433
sink.flush()497
sink.flush()434
if err != nil {498
if err != nil {435
return mask, fmt.Errorf("container ended with an error: %w", err)499
return fmt.Errorf("container ended with an error: %w", err)436
}500
}437
if err := r.collect(ctx, job, work); err != nil {501
return nil438
return mask, err439
}440
if err := r.saveCache(repo.ID, job, work); err != nil {441
return mask, err442
}443
return mask, nil444
}502
}445
503
446
// placeNote zijn de vaste regels in de sectie Preparing environment.504
// placeNote zijn de vaste regels in de sectie Preparing environment.@@ -985,6 +1043,7 @@ type logSink struct {985
mu sync.Mutex1043
mu sync.Mutex986
store *store.Store1044
store *store.Store987
jobID int641045
jobID int641046
out func(string)988
secrets []string1047
secrets []string989
partial string1048
partial string990
current string1049
current string@@ -1028,6 +1087,10 @@ func (l *logSink) emit(line string) {1028
l.opened = time.Now()1087
l.opened = time.Now()1029
}1088
}1030
}1089
}1090
if l.out != nil {1091
l.out(stamp(line))1092
return1093
}1031
_ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))1094
_ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))1032
}1095
}1033
1096
@@ -1,6 +1,7 @@
1
1
package server
2
2
3
3
import (
4
"context"
4
5
"net/http"
5
6
"strconv"
6
7
"strings"
@@ -345,11 +346,20 @@ func (s *Server) pipeJSON(repo store.Repo, p store.Pipeline) map[string]any {
345
346
}
346
347
}
347
348
348
func jobJSON(j store.Job) map[string]any {
349
return map[string]any{
349
func (s *Server) jobJSON(ctx context.Context, j store.Job) map[string]any {
350
out := map[string]any{
350
351
"id": j.ID, "name": j.Name, "stage": j.Stage, "status": j.Status, "when": j.When,
351
352
"allow_failure": j.AllowFailure, "started_at": rfcTime(j.StartedAt), "finished_at": rfcTime(j.FinishedAt),
353
"runner": nil,
354
}
355
if jr := s.Store.JobRun(ctx, j.ID); jr.RunnerName != "" {
356
kind := "builtin"
357
if jr.RunnerID > 0 {
358
kind = "remote"
359
}
360
out["runner"] = map[string]any{"id": jr.RunnerID, "name": jr.RunnerName, "kind": kind}
352
361
}
362
return out
353
363
}
354
364
355
365
func (s *Server) apiPipelines(w http.ResponseWriter, r *http.Request) {
@@ -488,7 +498,7 @@ func (s *Server) apiJobs(w http.ResponseWriter, r *http.Request) {
488
498
}
489
499
rows := make([]map[string]any, 0, len(jobs))
490
500
for _, j := range jobs {
491
rows = append(rows, jobJSON(j))
501
rows = append(rows, s.jobJSON(r.Context(), j))
492
502
}
493
503
writeJSON(w, http.StatusOK, rows)
494
504
}
@@ -498,7 +508,7 @@ func (s *Server) apiJob(w http.ResponseWriter, r *http.Request) {
498
508
if !ok {
499
509
return
500
510
}
501
writeJSON(w, http.StatusOK, jobJSON(job))
511
writeJSON(w, http.StatusOK, s.jobJSON(r.Context(), job))
502
512
}
503
513
504
514
func (s *Server) apiJobLog(w http.ResponseWriter, r *http.Request) {
@@ -520,7 +530,7 @@ func (s *Server) apiJobAction(w http.ResponseWriter, r *http.Request, action str
520
530
if !s.apiOK(w, r, err) {
521
531
return
522
532
}
523
writeJSON(w, http.StatusOK, jobJSON(updated))
533
writeJSON(w, http.StatusOK, s.jobJSON(r.Context(), updated))
524
534
}
525
535
526
536
func (s *Server) apiPlayJob(w http.ResponseWriter, r *http.Request) { s.apiJobAction(w, r, "play") }
@@ -1,6 +1,7 @@1
package server1
package server2
2
3
import (3
import (4
"context"4
"net/http"5
"net/http"5
"strconv"6
"strconv"6
"strings"7
"strings"@@ -345,11 +346,20 @@ func (s *Server) pipeJSON(repo store.Repo, p store.Pipeline) map[string]any {345
}346
}346
}347
}347
348
348
func jobJSON(j store.Job) map[string]any {349
func (s *Server) jobJSON(ctx context.Context, j store.Job) map[string]any {349
return map[string]any{350
out := map[string]any{350
"id": j.ID, "name": j.Name, "stage": j.Stage, "status": j.Status, "when": j.When,351
"id": j.ID, "name": j.Name, "stage": j.Stage, "status": j.Status, "when": j.When,351
"allow_failure": j.AllowFailure, "started_at": rfcTime(j.StartedAt), "finished_at": rfcTime(j.FinishedAt),352
"allow_failure": j.AllowFailure, "started_at": rfcTime(j.StartedAt), "finished_at": rfcTime(j.FinishedAt),353
"runner": nil,354
}355
if jr := s.Store.JobRun(ctx, j.ID); jr.RunnerName != "" {356
kind := "builtin"357
if jr.RunnerID > 0 {358
kind = "remote"359
}360
out["runner"] = map[string]any{"id": jr.RunnerID, "name": jr.RunnerName, "kind": kind}352
}361
}362
return out353
}363
}354
364
355
func (s *Server) apiPipelines(w http.ResponseWriter, r *http.Request) {365
func (s *Server) apiPipelines(w http.ResponseWriter, r *http.Request) {@@ -488,7 +498,7 @@ func (s *Server) apiJobs(w http.ResponseWriter, r *http.Request) {488
}498
}489
rows := make([]map[string]any, 0, len(jobs))499
rows := make([]map[string]any, 0, len(jobs))490
for _, j := range jobs {500
for _, j := range jobs {491
rows = append(rows, jobJSON(j))501
rows = append(rows, s.jobJSON(r.Context(), j))492
}502
}493
writeJSON(w, http.StatusOK, rows)503
writeJSON(w, http.StatusOK, rows)494
}504
}@@ -498,7 +508,7 @@ func (s *Server) apiJob(w http.ResponseWriter, r *http.Request) {498
if !ok {508
if !ok {499
return509
return500
}510
}501
writeJSON(w, http.StatusOK, jobJSON(job))511
writeJSON(w, http.StatusOK, s.jobJSON(r.Context(), job))502
}512
}503
513
504
func (s *Server) apiJobLog(w http.ResponseWriter, r *http.Request) {514
func (s *Server) apiJobLog(w http.ResponseWriter, r *http.Request) {@@ -520,7 +530,7 @@ func (s *Server) apiJobAction(w http.ResponseWriter, r *http.Request, action str520
if !s.apiOK(w, r, err) {530
if !s.apiOK(w, r, err) {521
return531
return522
}532
}523
writeJSON(w, http.StatusOK, jobJSON(updated))533
writeJSON(w, http.StatusOK, s.jobJSON(r.Context(), updated))524
}534
}525
535
526
func (s *Server) apiPlayJob(w http.ResponseWriter, r *http.Request) { s.apiJobAction(w, r, "play") }536
func (s *Server) apiPlayJob(w http.ResponseWriter, r *http.Request) { s.apiJobAction(w, r, "play") }@@ -160,6 +160,7 @@ func (s *Server) account(w http.ResponseWriter, r *http.Request) {
160
160
if v.NewToken != "" {
161
161
http.SetCookie(w, &http.Cookie{Name: "bluegit_token_once", Value: "", Path: "/", MaxAge: -1})
162
162
}
163
s.fillUserRunners(w, r, &v, u)
163
164
s.render(w, "account", v)
164
165
}
165
166
@@ -160,6 +160,7 @@ func (s *Server) account(w http.ResponseWriter, r *http.Request) {160
if v.NewToken != "" {160
if v.NewToken != "" {161
http.SetCookie(w, &http.Cookie{Name: "bluegit_token_once", Value: "", Path: "/", MaxAge: -1})161
http.SetCookie(w, &http.Cookie{Name: "bluegit_token_once", Value: "", Path: "/", MaxAge: -1})162
}162
}163
s.fillUserRunners(w, r, &v, u)163
s.render(w, "account", v)164
s.render(w, "account", v)164
}165
}165
166
@@ -28,6 +28,13 @@ func (s *Server) gitHTTP(w http.ResponseWriter, r *http.Request) {
28
28
}
29
29
service := r.URL.Query().Get("service")
30
30
write := service == "git-receive-pack" || strings.HasSuffix(r.URL.Path, "/git-receive-pack")
31
if !write && s.runnerMayClone(r, repo.ID) {
32
rest := strings.TrimPrefix(r.URL.Path, "/"+owner+"/"+repoSeg)
33
if err := gitrepo.ServeSmartHTTP(w, r, s.Git, gitrepo.ProjectRoot(s.StoragePath()), gitrepo.PathInfo(owner, repoSeg, rest), nil); err != nil && s.Log != nil {
34
s.Log.Error("git http", "err", err, "path", r.URL.Path)
35
}
36
return
37
}
31
38
user, scopes, authed := s.gitUser(r)
32
39
var role access.Role
33
40
if authed {
@@ -102,6 +109,21 @@ func (s *Server) gitUser(r *http.Request) (store.User, access.Scopes, bool) {
102
109
return auth.User, access.StoredScopes(auth.Token.Scopes), true
103
110
}
104
111
112
// runnerMayClone: een aparte runner (bgr_-token) mag alleen lezen uit de
113
// repo van een job die hij nu draait. Job-tokens krijgen geen git-toegang.
114
func (s *Server) runnerMayClone(r *http.Request, repoID int64) bool {
115
h := r.Header.Get("Authorization")
116
if !strings.HasPrefix(strings.ToLower(h), "bearer ") {
117
return false
118
}
119
tok := strings.TrimSpace(h[7:])
120
if !strings.HasPrefix(tok, "bgr_") {
121
return false
122
}
123
rn, err := s.Store.RunnerByToken(r.Context(), tok)
124
return err == nil && s.Store.RunnerRunsRepo(r.Context(), rn.ID, repoID)
125
}
126
105
127
func (s *Server) askAuth(w http.ResponseWriter) {
106
128
w.Header().Set("WWW-Authenticate", `Basic realm="BlueGit"`)
107
129
http.Error(w, "aanmelden met een toegangstoken als wachtwoord", http.StatusUnauthorized)
@@ -28,6 +28,13 @@ func (s *Server) gitHTTP(w http.ResponseWriter, r *http.Request) {28
}28
}29
service := r.URL.Query().Get("service")29
service := r.URL.Query().Get("service")30
write := service == "git-receive-pack" || strings.HasSuffix(r.URL.Path, "/git-receive-pack")30
write := service == "git-receive-pack" || strings.HasSuffix(r.URL.Path, "/git-receive-pack")31
if !write && s.runnerMayClone(r, repo.ID) {32
rest := strings.TrimPrefix(r.URL.Path, "/"+owner+"/"+repoSeg)33
if err := gitrepo.ServeSmartHTTP(w, r, s.Git, gitrepo.ProjectRoot(s.StoragePath()), gitrepo.PathInfo(owner, repoSeg, rest), nil); err != nil && s.Log != nil {34
s.Log.Error("git http", "err", err, "path", r.URL.Path)35
}36
return37
}31
user, scopes, authed := s.gitUser(r)38
user, scopes, authed := s.gitUser(r)32
var role access.Role39
var role access.Role33
if authed {40
if authed {@@ -102,6 +109,21 @@ func (s *Server) gitUser(r *http.Request) (store.User, access.Scopes, bool) {102
return auth.User, access.StoredScopes(auth.Token.Scopes), true109
return auth.User, access.StoredScopes(auth.Token.Scopes), true103
}110
}104
111
112
// runnerMayClone: een aparte runner (bgr_-token) mag alleen lezen uit de113
// repo van een job die hij nu draait. Job-tokens krijgen geen git-toegang.114
func (s *Server) runnerMayClone(r *http.Request, repoID int64) bool {115
h := r.Header.Get("Authorization")116
if !strings.HasPrefix(strings.ToLower(h), "bearer ") {117
return false118
}119
tok := strings.TrimSpace(h[7:])120
if !strings.HasPrefix(tok, "bgr_") {121
return false122
}123
rn, err := s.Store.RunnerByToken(r.Context(), tok)124
return err == nil && s.Store.RunnerRunsRepo(r.Context(), rn.ID, repoID)125
}126
105
func (s *Server) askAuth(w http.ResponseWriter) {127
func (s *Server) askAuth(w http.ResponseWriter) {106
w.Header().Set("WWW-Authenticate", `Basic realm="BlueGit"`)128
w.Header().Set("WWW-Authenticate", `Basic realm="BlueGit"`)107
http.Error(w, "aanmelden met een toegangstoken als wachtwoord", http.StatusUnauthorized)129
http.Error(w, "aanmelden met een toegangstoken als wachtwoord", http.StatusUnauthorized)@@ -62,6 +62,7 @@ type JobLink struct {
62
62
// JobPage is de console van één job.
63
63
type JobPage struct {
64
64
Job store.Job
65
Runner string
65
66
Pipe store.Pipeline
66
67
Name string
67
68
Status string
@@ -598,7 +599,8 @@ func (s *Server) jobPage(r *http.Request, repo store.Repo, p store.Pipeline, job
598
599
source = p.Source
599
600
}
600
601
return &JobPage{
601
Job: job, Pipe: p, Name: job.Name, Status: status, Label: s.tr(r, "pipe."+status),
602
Runner: s.Store.JobRun(r.Context(), job.ID).RunnerName,
603
Job: job, Pipe: p, Name: job.Name, Status: status, Label: s.tr(r, "pipe."+status),
602
604
Actor: actor, ActorURL: actorURL, Avatar: avatar, Initial: initial, When: i18n.Ago(lang, p.CreatedAt.Unix()),
603
605
Duration: dur, Finished: finished, Queued: queued, Timeout: timeout, Source: source,
604
606
Ref: name, RefKind: kind, Short: shortSHA(p.CommitSHA), Title: p.CommitTitle, Stage: job.Stage,
@@ -62,6 +62,7 @@ type JobLink struct {62
// JobPage is de console van één job.62
// JobPage is de console van één job.63
type JobPage struct {63
type JobPage struct {64
Job store.Job64
Job store.Job65
Runner string65
Pipe store.Pipeline66
Pipe store.Pipeline66
Name string67
Name string67
Status string68
Status string@@ -598,7 +599,8 @@ func (s *Server) jobPage(r *http.Request, repo store.Repo, p store.Pipeline, job598
source = p.Source599
source = p.Source599
}600
}600
return &JobPage{601
return &JobPage{601
Job: job, Pipe: p, Name: job.Name, Status: status, Label: s.tr(r, "pipe."+status),602
Runner: s.Store.JobRun(r.Context(), job.ID).RunnerName,603
Job: job, Pipe: p, Name: job.Name, Status: status, Label: s.tr(r, "pipe."+status),602
Actor: actor, ActorURL: actorURL, Avatar: avatar, Initial: initial, When: i18n.Ago(lang, p.CreatedAt.Unix()),604
Actor: actor, ActorURL: actorURL, Avatar: avatar, Initial: initial, When: i18n.Ago(lang, p.CreatedAt.Unix()),603
Duration: dur, Finished: finished, Queued: queued, Timeout: timeout, Source: source,605
Duration: dur, Finished: finished, Queued: queued, Timeout: timeout, Source: source,604
Ref: name, RefKind: kind, Short: shortSHA(p.CommitSHA), Title: p.CommitTitle, Stage: job.Stage,606
Ref: name, RefKind: kind, Short: shortSHA(p.CommitSHA), Title: p.CommitTitle, Stage: job.Stage,@@ -0,0 +1,320 @@
1
package server
2
3
import (
4
"context"
5
"encoding/json"
6
"io"
7
"log/slog"
8
"net/http"
9
"net/http/httptest"
10
"os"
11
"os/exec"
12
"path/filepath"
13
"runtime"
14
"strings"
15
"testing"
16
"time"
17
18
"gitlab.com/Timfa/bluegit/internal/auth"
19
"gitlab.com/Timfa/bluegit/internal/config"
20
"gitlab.com/Timfa/bluegit/internal/gitrepo"
21
"gitlab.com/Timfa/bluegit/internal/runner"
22
"gitlab.com/Timfa/bluegit/internal/store"
23
)
24
25
// fakeDocker voert "docker run ... sh /workspace/.bluegit-job.sh" lokaal
26
// uit in de gemounte werkmap. Andere docker-commando's slagen stil.
27
const fakeDocker = `#!/bin/sh
28
[ "$1" = run ] || exit 0
29
shift
30
work=
31
while [ $# -gt 0 ]; do
32
case "$1" in
33
-v) work="${2%%:/workspace}"; shift 2;;
34
--network|-w|-e|--entrypoint|--name|--network-alias) shift 2;;
35
--rm|-d) shift;;
36
*) break;;
37
esac
38
done
39
cd "$work" || exit 1
40
sed "s#^cd /workspace\$#cd \"$work\"#" .bluegit-job.sh > .fake-job.sh
41
exec sh .fake-job.sh
42
`
43
44
type ciEnv struct {
45
t *testing.T
46
st *store.Store
47
srv *Server
48
ts *httptest.Server
49
rn *runner.Runner
50
repo store.Repo
51
sha string
52
docker string
53
user store.User
54
}
55
56
func newCIEnv(t *testing.T) *ciEnv {
57
t.Helper()
58
if runtime.GOOS == "windows" {
59
t.Skip("sh nodig")
60
}
61
for _, bin := range []string{"git", "sh", "sed"} {
62
if _, err := exec.LookPath(bin); err != nil {
63
t.Skip(bin + " ontbreekt")
64
}
65
}
66
storage := t.TempDir()
67
st, err := store.Open(filepath.Join(storage, "bluegit.db"), bytes32())
68
if err != nil {
69
t.Fatal(err)
70
}
71
t.Cleanup(func() { _ = st.Close() })
72
docker := filepath.Join(t.TempDir(), "docker")
73
if err := os.WriteFile(docker, []byte(fakeDocker), 0o755); err != nil {
74
t.Fatal(err)
75
}
76
cfg := config.Config{
77
Listen: "127.0.0.1:0", StoragePath: storage,
78
Auth: config.Auth{Mode: "dev", DevUser: "tim", DevAuthID: 1, UserAgent: "test", BaseURL: "https://auth.timfalken.com"},
79
Git: config.Git{Binary: "git", DefaultBranch: "master"},
80
CI: config.CI{Enabled: true, Docker: docker},
81
}
82
srv := &Server{Cfg: cfg, Store: st, Auth: auth.Dev{Username: "tim", AuthID: 1}, Dev: true, Git: "git", HookBin: "/bin/true"}
83
ts := httptest.NewUnstartedServer(nil)
84
ts.Start()
85
t.Cleanup(ts.Close)
86
srv.Cfg.BaseURL = ts.URL
87
rn := &runner.Runner{
88
Store: st, GitBin: "git", Docker: docker, Storage: storage, BaseURL: ts.URL, Network: "bridge",
89
Name: "BlueGit runner", Timeout: time.Minute, Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
90
}
91
srv.Runner = rn
92
ts.Config.Handler = srv.Handler()
93
94
ctx := context.Background()
95
tim, err := st.UpsertUser(ctx, 1, "tim", "", false)
96
if err != nil {
97
t.Fatal(err)
98
}
99
repo, err := st.CreateRepo(ctx, "user", tim.ID, "app", "", "master", "private")
100
if err != nil {
101
t.Fatal(err)
102
}
103
bare := gitrepo.Dir(storage, repo.OwnerPath, repo.Name)
104
if err := gitrepo.Init("git", bare, "master", "/bin/true"); err != nil {
105
t.Fatal(err)
106
}
107
src := t.TempDir()
108
run := func(dir string, args ...string) string {
109
cmd := exec.Command("git", args...)
110
cmd.Dir = dir
111
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x")
112
out, err := cmd.CombinedOutput()
113
if err != nil {
114
t.Fatalf("git %v: %v %s", args, err, out)
115
}
116
return strings.TrimSpace(string(out))
117
}
118
run(src, "init", "-q", "-b", "master")
119
_ = os.WriteFile(filepath.Join(src, "README.md"), []byte("hello from the repo\n"), 0o644)
120
run(src, "add", ".")
121
run(src, "commit", "-qm", "init")
122
run(src, "push", "-q", bare, "master")
123
sha := run(src, "rev-parse", "HEAD")
124
return &ciEnv{t: t, st: st, srv: srv, ts: ts, rn: rn, repo: repo, sha: sha, docker: docker, user: tim}
125
}
126
127
func (e *ciEnv) pipeline() int64 {
128
e.t.Helper()
129
id, err := e.st.CreatePipeline(context.Background(), e.repo.ID, e.sha, "refs/heads/master", "push", "tim", "init", []store.NewJob{
130
{Name: "build", Stage: "build", Position: 0, Image: "alpine:3.20", Script: "cat README.md\necho built-artifact > out.txt\necho \"token-set=${CI_JOB_TOKEN:+yes}\"", ArtifactPaths: []string{"out.txt"}},
131
{Name: "test", Stage: "test", Position: 1, Image: "alpine:3.20", Script: "echo got: $(cat out.txt)", Needs: []string{"build"}, NeedsSet: true},
132
})
133
if err != nil {
134
e.t.Fatal(err)
135
}
136
return id
137
}
138
139
// tick draait de planner tot de pipeline klaar is.
140
func (e *ciEnv) wait(pid int64, limit time.Duration) store.Pipeline {
141
e.t.Helper()
142
ctx := context.Background()
143
deadline := time.Now().Add(limit)
144
for time.Now().Before(deadline) {
145
e.rn.Tick(ctx)
146
p, err := e.st.PipelineByID(ctx, e.repo.ID, pid)
147
if err != nil {
148
e.t.Fatal(err)
149
}
150
if p.Status == "success" || p.Status == "failed" || p.Status == "canceled" {
151
return p
152
}
153
time.Sleep(200 * time.Millisecond)
154
}
155
jobs, _ := e.st.JobsByPipeline(ctx, pid)
156
for _, j := range jobs {
157
e.t.Logf("job %s %s\n%s", j.Name, j.Status, j.Log)
158
}
159
e.t.Fatalf("pipeline %d not finished", pid)
160
return store.Pipeline{}
161
}
162
163
func (e *ciEnv) checkJobs(pid int64, wantRunner string, remote bool) {
164
e.t.Helper()
165
ctx := context.Background()
166
jobs, _ := e.st.JobsByPipeline(ctx, pid)
167
if len(jobs) != 2 {
168
e.t.Fatalf("jobs: %d", len(jobs))
169
}
170
for _, j := range jobs {
171
if j.Status != "success" {
172
e.t.Fatalf("job %s: %s\n%s", j.Name, j.Status, j.Log)
173
}
174
jr := e.st.JobRun(ctx, j.ID)
175
if jr.RunnerName != wantRunner || (jr.RunnerID > 0) != remote {
176
e.t.Fatalf("job %s ran on %+v, want %s remote=%v", j.Name, jr, wantRunner, remote)
177
}
178
}
179
if !strings.Contains(jobs[0].Log, "hello from the repo") || !strings.Contains(jobs[0].Log, "token-set=yes") {
180
e.t.Fatalf("build log:\n%s", jobs[0].Log)
181
}
182
if strings.Contains(jobs[0].Log, "bgj_") {
183
e.t.Fatalf("job token leaked in log:\n%s", jobs[0].Log)
184
}
185
if !strings.Contains(jobs[1].Log, "got: built-artifact") {
186
e.t.Fatalf("test log (artifact niet doorgegeven):\n%s", jobs[1].Log)
187
}
188
if !strings.Contains(jobs[0].Log, wantRunner) {
189
e.t.Fatalf("runner name not in log:\n%s", jobs[0].Log)
190
}
191
// API toont de runner.
192
if js := e.srv.jobJSON(ctx, jobs[0]); js["runner"] == nil {
193
e.t.Fatal("jobJSON without runner")
194
}
195
}
196
197
// De ingebouwde runner werkt nog zoals voorheen.
198
func TestBuiltinRunnerStillWorks(t *testing.T) {
199
e := newCIEnv(t)
200
pid := e.pipeline()
201
p := e.wait(pid, 60*time.Second)
202
if p.Status != "success" {
203
t.Fatalf("pipeline %s", p.Status)
204
}
205
e.checkJobs(pid, "BlueGit runner", false)
206
}
207
208
func buildRunnerBinary(t *testing.T) string {
209
t.Helper()
210
if _, err := exec.LookPath("go"); err != nil {
211
t.Skip("go ontbreekt")
212
}
213
bin := filepath.Join(t.TempDir(), "bluegit-runner")
214
cmd := exec.Command("go", "build", "-o", bin, "../../cmd/bluegit-runner")
215
if out, err := cmd.CombinedOutput(); err != nil {
216
t.Fatalf("build bluegit-runner: %v %s", err, out)
217
}
218
return bin
219
}
220
221
// De aparte bluegit-runner registreert zich echt en draait de jobs.
222
func TestSeparateRunnerEndToEnd(t *testing.T) {
223
e := newCIEnv(t)
224
bin := buildRunnerBinary(t)
225
ctx := context.Background()
226
if err := e.st.SetUserRunnerPref(ctx, e.user.ID, store.RunnerPrefOwnFirst); err != nil {
227
t.Fatal(err)
228
}
229
reg, err := e.st.CreateRunnerRegToken(ctx, "user", e.user.ID, e.user.ID)
230
if err != nil {
231
t.Fatal(err)
232
}
233
cfgDir := t.TempDir()
234
cfgPath := filepath.Join(cfgDir, "config.yml")
235
work := t.TempDir()
236
if err := os.WriteFile(cfgPath, []byte("docker: "+e.docker+"\nwork_dir: "+work+"\n"), 0o600); err != nil {
237
t.Fatal(err)
238
}
239
out, err := exec.Command(bin, "register", "--url", e.ts.URL, "--token", reg, "--name", "tim-laptop", "--config", cfgPath).CombinedOutput()
240
if err != nil {
241
t.Fatalf("register: %v %s", err, out)
242
}
243
cfgB, _ := os.ReadFile(cfgPath)
244
if !strings.Contains(string(cfgB), "bgr_") || !strings.Contains(string(cfgB), e.docker) {
245
t.Fatalf("config not written correctly")
246
}
247
// Het registratietoken is eenmalig.
248
if out, err := exec.Command(bin, "register", "--url", e.ts.URL, "--token", reg, "--config", filepath.Join(cfgDir, "b.yml")).CombinedOutput(); err == nil {
249
t.Fatalf("second registration succeeded: %s", out)
250
}
251
// Zonder lopende job mag het runner-token niet clonen.
252
var tok string
253
for _, line := range strings.Split(string(cfgB), "\n") {
254
if strings.HasPrefix(line, "token: ") {
255
tok = strings.TrimSpace(strings.TrimPrefix(line, "token: "))
256
}
257
}
258
req, _ := http.NewRequest(http.MethodGet, e.ts.URL+"/tim/app.git/info/refs?service=git-upload-pack", nil)
259
req.Header.Set("Authorization", "Bearer "+tok)
260
if res, err := http.DefaultClient.Do(req); err != nil || res.StatusCode == http.StatusOK {
261
t.Fatalf("runner token cloned without a job: %v", err)
262
}
263
264
runCtx, stop := context.WithCancel(ctx)
265
defer stop()
266
cmd := exec.CommandContext(runCtx, bin, "run", "--config", cfgPath)
267
var logs strings.Builder
268
cmd.Stdout, cmd.Stderr = &logs, &logs
269
if err := cmd.Start(); err != nil {
270
t.Fatal(err)
271
}
272
defer func() { stop(); _ = cmd.Wait(); t.Logf("runner output:\n%s", logs.String()) }()
273
274
pid := e.pipeline()
275
p := e.wait(pid, 90*time.Second)
276
if p.Status != "success" {
277
jobs, _ := e.st.JobsByPipeline(ctx, pid)
278
for _, j := range jobs {
279
t.Logf("%s %s\n%s", j.Name, j.Status, j.Log)
280
}
281
t.Fatalf("pipeline %s", p.Status)
282
}
283
e.checkJobs(pid, "tim-laptop", true)
284
rs, _ := e.st.ListRunners(ctx, "user", e.user.ID)
285
if len(rs) != 1 || !rs[0].Online(time.Now()) {
286
t.Fatalf("runner status: %+v", rs)
287
}
288
// Artifact van de remote job staat op de server.
289
jobs, _ := e.st.JobsByPipeline(ctx, pid)
290
if path, err := e.st.ArtifactPath(ctx, pid, jobs[0].Name, "out.txt"); err != nil {
291
t.Fatal(err)
292
} else if b, _ := os.ReadFile(path); strings.TrimSpace(string(b)) != "built-artifact" {
293
t.Fatalf("artifact: %q", b)
294
}
295
// Beheer-UI en API: de jobpagina-JSON noemt de runner.
296
js, _ := json.Marshal(e.srv.jobJSON(ctx, jobs[1]))
297
if !strings.Contains(string(js), `"tim-laptop"`) || !strings.Contains(string(js), `"remote"`) {
298
t.Fatalf("jobJSON: %s", js)
299
}
300
}
301
302
// Een runner die de job niet oppakt: na de wachttijd neemt de BlueGit-runner het over.
303
func TestSeparateRunnerFallback(t *testing.T) {
304
e := newCIEnv(t)
305
ctx := context.Background()
306
old := runner.OfferTimeout
307
runner.OfferTimeout = time.Second
308
t.Cleanup(func() { runner.OfferTimeout = old })
309
_ = e.st.SetUserRunnerPref(ctx, e.user.ID, store.RunnerPrefOwnFirst)
310
reg, _ := e.st.CreateRunnerRegToken(ctx, "user", e.user.ID, e.user.ID)
311
if _, _, err := e.st.RegisterRunner(ctx, reg, "silent", "test"); err != nil {
312
t.Fatal(err)
313
}
314
pid := e.pipeline()
315
p := e.wait(pid, 60*time.Second)
316
if p.Status != "success" {
317
t.Fatalf("pipeline %s", p.Status)
318
}
319
e.checkJobs(pid, "BlueGit runner", false)
320
}
@@ -0,0 +1,320 @@1
package server2
3
import (4
"context"5
"encoding/json"6
"io"7
"log/slog"8
"net/http"9
"net/http/httptest"10
"os"11
"os/exec"12
"path/filepath"13
"runtime"14
"strings"15
"testing"16
"time"17
18
"gitlab.com/Timfa/bluegit/internal/auth"19
"gitlab.com/Timfa/bluegit/internal/config"20
"gitlab.com/Timfa/bluegit/internal/gitrepo"21
"gitlab.com/Timfa/bluegit/internal/runner"22
"gitlab.com/Timfa/bluegit/internal/store"23
)24
25
// fakeDocker voert "docker run ... sh /workspace/.bluegit-job.sh" lokaal26
// uit in de gemounte werkmap. Andere docker-commando's slagen stil.27
const fakeDocker = `#!/bin/sh28
[ "$1" = run ] || exit 029
shift30
work=31
while [ $# -gt 0 ]; do32
case "$1" in33
-v) work="${2%%:/workspace}"; shift 2;;34
--network|-w|-e|--entrypoint|--name|--network-alias) shift 2;;35
--rm|-d) shift;;36
*) break;;37
esac38
done39
cd "$work" || exit 140
sed "s#^cd /workspace\$#cd \"$work\"#" .bluegit-job.sh > .fake-job.sh41
exec sh .fake-job.sh42
`43
44
type ciEnv struct {45
t *testing.T46
st *store.Store47
srv *Server48
ts *httptest.Server49
rn *runner.Runner50
repo store.Repo51
sha string52
docker string53
user store.User54
}55
56
func newCIEnv(t *testing.T) *ciEnv {57
t.Helper()58
if runtime.GOOS == "windows" {59
t.Skip("sh nodig")60
}61
for _, bin := range []string{"git", "sh", "sed"} {62
if _, err := exec.LookPath(bin); err != nil {63
t.Skip(bin + " ontbreekt")64
}65
}66
storage := t.TempDir()67
st, err := store.Open(filepath.Join(storage, "bluegit.db"), bytes32())68
if err != nil {69
t.Fatal(err)70
}71
t.Cleanup(func() { _ = st.Close() })72
docker := filepath.Join(t.TempDir(), "docker")73
if err := os.WriteFile(docker, []byte(fakeDocker), 0o755); err != nil {74
t.Fatal(err)75
}76
cfg := config.Config{77
Listen: "127.0.0.1:0", StoragePath: storage,78
Auth: config.Auth{Mode: "dev", DevUser: "tim", DevAuthID: 1, UserAgent: "test", BaseURL: "https://auth.timfalken.com"},79
Git: config.Git{Binary: "git", DefaultBranch: "master"},80
CI: config.CI{Enabled: true, Docker: docker},81
}82
srv := &Server{Cfg: cfg, Store: st, Auth: auth.Dev{Username: "tim", AuthID: 1}, Dev: true, Git: "git", HookBin: "/bin/true"}83
ts := httptest.NewUnstartedServer(nil)84
ts.Start()85
t.Cleanup(ts.Close)86
srv.Cfg.BaseURL = ts.URL87
rn := &runner.Runner{88
Store: st, GitBin: "git", Docker: docker, Storage: storage, BaseURL: ts.URL, Network: "bridge",89
Name: "BlueGit runner", Timeout: time.Minute, Log: slog.New(slog.NewTextHandler(io.Discard, nil)),90
}91
srv.Runner = rn92
ts.Config.Handler = srv.Handler()93
94
ctx := context.Background()95
tim, err := st.UpsertUser(ctx, 1, "tim", "", false)96
if err != nil {97
t.Fatal(err)98
}99
repo, err := st.CreateRepo(ctx, "user", tim.ID, "app", "", "master", "private")100
if err != nil {101
t.Fatal(err)102
}103
bare := gitrepo.Dir(storage, repo.OwnerPath, repo.Name)104
if err := gitrepo.Init("git", bare, "master", "/bin/true"); err != nil {105
t.Fatal(err)106
}107
src := t.TempDir()108
run := func(dir string, args ...string) string {109
cmd := exec.Command("git", args...)110
cmd.Dir = dir111
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@x", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@x")112
out, err := cmd.CombinedOutput()113
if err != nil {114
t.Fatalf("git %v: %v %s", args, err, out)115
}116
return strings.TrimSpace(string(out))117
}118
run(src, "init", "-q", "-b", "master")119
_ = os.WriteFile(filepath.Join(src, "README.md"), []byte("hello from the repo\n"), 0o644)120
run(src, "add", ".")121
run(src, "commit", "-qm", "init")122
run(src, "push", "-q", bare, "master")123
sha := run(src, "rev-parse", "HEAD")124
return &ciEnv{t: t, st: st, srv: srv, ts: ts, rn: rn, repo: repo, sha: sha, docker: docker, user: tim}125
}126
127
func (e *ciEnv) pipeline() int64 {128
e.t.Helper()129
id, err := e.st.CreatePipeline(context.Background(), e.repo.ID, e.sha, "refs/heads/master", "push", "tim", "init", []store.NewJob{130
{Name: "build", Stage: "build", Position: 0, Image: "alpine:3.20", Script: "cat README.md\necho built-artifact > out.txt\necho \"token-set=${CI_JOB_TOKEN:+yes}\"", ArtifactPaths: []string{"out.txt"}},131
{Name: "test", Stage: "test", Position: 1, Image: "alpine:3.20", Script: "echo got: $(cat out.txt)", Needs: []string{"build"}, NeedsSet: true},132
})133
if err != nil {134
e.t.Fatal(err)135
}136
return id137
}138
139
// tick draait de planner tot de pipeline klaar is.140
func (e *ciEnv) wait(pid int64, limit time.Duration) store.Pipeline {141
e.t.Helper()142
ctx := context.Background()143
deadline := time.Now().Add(limit)144
for time.Now().Before(deadline) {145
e.rn.Tick(ctx)146
p, err := e.st.PipelineByID(ctx, e.repo.ID, pid)147
if err != nil {148
e.t.Fatal(err)149
}150
if p.Status == "success" || p.Status == "failed" || p.Status == "canceled" {151
return p152
}153
time.Sleep(200 * time.Millisecond)154
}155
jobs, _ := e.st.JobsByPipeline(ctx, pid)156
for _, j := range jobs {157
e.t.Logf("job %s %s\n%s", j.Name, j.Status, j.Log)158
}159
e.t.Fatalf("pipeline %d not finished", pid)160
return store.Pipeline{}161
}162
163
func (e *ciEnv) checkJobs(pid int64, wantRunner string, remote bool) {164
e.t.Helper()165
ctx := context.Background()166
jobs, _ := e.st.JobsByPipeline(ctx, pid)167
if len(jobs) != 2 {168
e.t.Fatalf("jobs: %d", len(jobs))169
}170
for _, j := range jobs {171
if j.Status != "success" {172
e.t.Fatalf("job %s: %s\n%s", j.Name, j.Status, j.Log)173
}174
jr := e.st.JobRun(ctx, j.ID)175
if jr.RunnerName != wantRunner || (jr.RunnerID > 0) != remote {176
e.t.Fatalf("job %s ran on %+v, want %s remote=%v", j.Name, jr, wantRunner, remote)177
}178
}179
if !strings.Contains(jobs[0].Log, "hello from the repo") || !strings.Contains(jobs[0].Log, "token-set=yes") {180
e.t.Fatalf("build log:\n%s", jobs[0].Log)181
}182
if strings.Contains(jobs[0].Log, "bgj_") {183
e.t.Fatalf("job token leaked in log:\n%s", jobs[0].Log)184
}185
if !strings.Contains(jobs[1].Log, "got: built-artifact") {186
e.t.Fatalf("test log (artifact niet doorgegeven):\n%s", jobs[1].Log)187
}188
if !strings.Contains(jobs[0].Log, wantRunner) {189
e.t.Fatalf("runner name not in log:\n%s", jobs[0].Log)190
}191
// API toont de runner.192
if js := e.srv.jobJSON(ctx, jobs[0]); js["runner"] == nil {193
e.t.Fatal("jobJSON without runner")194
}195
}196
197
// De ingebouwde runner werkt nog zoals voorheen.198
func TestBuiltinRunnerStillWorks(t *testing.T) {199
e := newCIEnv(t)200
pid := e.pipeline()201
p := e.wait(pid, 60*time.Second)202
if p.Status != "success" {203
t.Fatalf("pipeline %s", p.Status)204
}205
e.checkJobs(pid, "BlueGit runner", false)206
}207
208
func buildRunnerBinary(t *testing.T) string {209
t.Helper()210
if _, err := exec.LookPath("go"); err != nil {211
t.Skip("go ontbreekt")212
}213
bin := filepath.Join(t.TempDir(), "bluegit-runner")214
cmd := exec.Command("go", "build", "-o", bin, "../../cmd/bluegit-runner")215
if out, err := cmd.CombinedOutput(); err != nil {216
t.Fatalf("build bluegit-runner: %v %s", err, out)217
}218
return bin219
}220
221
// De aparte bluegit-runner registreert zich echt en draait de jobs.222
func TestSeparateRunnerEndToEnd(t *testing.T) {223
e := newCIEnv(t)224
bin := buildRunnerBinary(t)225
ctx := context.Background()226
if err := e.st.SetUserRunnerPref(ctx, e.user.ID, store.RunnerPrefOwnFirst); err != nil {227
t.Fatal(err)228
}229
reg, err := e.st.CreateRunnerRegToken(ctx, "user", e.user.ID, e.user.ID)230
if err != nil {231
t.Fatal(err)232
}233
cfgDir := t.TempDir()234
cfgPath := filepath.Join(cfgDir, "config.yml")235
work := t.TempDir()236
if err := os.WriteFile(cfgPath, []byte("docker: "+e.docker+"\nwork_dir: "+work+"\n"), 0o600); err != nil {237
t.Fatal(err)238
}239
out, err := exec.Command(bin, "register", "--url", e.ts.URL, "--token", reg, "--name", "tim-laptop", "--config", cfgPath).CombinedOutput()240
if err != nil {241
t.Fatalf("register: %v %s", err, out)242
}243
cfgB, _ := os.ReadFile(cfgPath)244
if !strings.Contains(string(cfgB), "bgr_") || !strings.Contains(string(cfgB), e.docker) {245
t.Fatalf("config not written correctly")246
}247
// Het registratietoken is eenmalig.248
if out, err := exec.Command(bin, "register", "--url", e.ts.URL, "--token", reg, "--config", filepath.Join(cfgDir, "b.yml")).CombinedOutput(); err == nil {249
t.Fatalf("second registration succeeded: %s", out)250
}251
// Zonder lopende job mag het runner-token niet clonen.252
var tok string253
for _, line := range strings.Split(string(cfgB), "\n") {254
if strings.HasPrefix(line, "token: ") {255
tok = strings.TrimSpace(strings.TrimPrefix(line, "token: "))256
}257
}258
req, _ := http.NewRequest(http.MethodGet, e.ts.URL+"/tim/app.git/info/refs?service=git-upload-pack", nil)259
req.Header.Set("Authorization", "Bearer "+tok)260
if res, err := http.DefaultClient.Do(req); err != nil || res.StatusCode == http.StatusOK {261
t.Fatalf("runner token cloned without a job: %v", err)262
}263
264
runCtx, stop := context.WithCancel(ctx)265
defer stop()266
cmd := exec.CommandContext(runCtx, bin, "run", "--config", cfgPath)267
var logs strings.Builder268
cmd.Stdout, cmd.Stderr = &logs, &logs269
if err := cmd.Start(); err != nil {270
t.Fatal(err)271
}272
defer func() { stop(); _ = cmd.Wait(); t.Logf("runner output:\n%s", logs.String()) }()273
274
pid := e.pipeline()275
p := e.wait(pid, 90*time.Second)276
if p.Status != "success" {277
jobs, _ := e.st.JobsByPipeline(ctx, pid)278
for _, j := range jobs {279
t.Logf("%s %s\n%s", j.Name, j.Status, j.Log)280
}281
t.Fatalf("pipeline %s", p.Status)282
}283
e.checkJobs(pid, "tim-laptop", true)284
rs, _ := e.st.ListRunners(ctx, "user", e.user.ID)285
if len(rs) != 1 || !rs[0].Online(time.Now()) {286
t.Fatalf("runner status: %+v", rs)287
}288
// Artifact van de remote job staat op de server.289
jobs, _ := e.st.JobsByPipeline(ctx, pid)290
if path, err := e.st.ArtifactPath(ctx, pid, jobs[0].Name, "out.txt"); err != nil {291
t.Fatal(err)292
} else if b, _ := os.ReadFile(path); strings.TrimSpace(string(b)) != "built-artifact" {293
t.Fatalf("artifact: %q", b)294
}295
// Beheer-UI en API: de jobpagina-JSON noemt de runner.296
js, _ := json.Marshal(e.srv.jobJSON(ctx, jobs[1]))297
if !strings.Contains(string(js), `"tim-laptop"`) || !strings.Contains(string(js), `"remote"`) {298
t.Fatalf("jobJSON: %s", js)299
}300
}301
302
// Een runner die de job niet oppakt: na de wachttijd neemt de BlueGit-runner het over.303
func TestSeparateRunnerFallback(t *testing.T) {304
e := newCIEnv(t)305
ctx := context.Background()306
old := runner.OfferTimeout307
runner.OfferTimeout = time.Second308
t.Cleanup(func() { runner.OfferTimeout = old })309
_ = e.st.SetUserRunnerPref(ctx, e.user.ID, store.RunnerPrefOwnFirst)310
reg, _ := e.st.CreateRunnerRegToken(ctx, "user", e.user.ID, e.user.ID)311
if _, _, err := e.st.RegisterRunner(ctx, reg, "silent", "test"); err != nil {312
t.Fatal(err)313
}314
pid := e.pipeline()315
p := e.wait(pid, 60*time.Second)316
if p.Status != "success" {317
t.Fatalf("pipeline %s", p.Status)318
}319
e.checkJobs(pid, "BlueGit runner", false)320
}@@ -0,0 +1,374 @@
1
package server
2
3
import (
4
"encoding/json"
5
"errors"
6
"io"
7
"net/http"
8
"os"
9
"strconv"
10
"strings"
11
"time"
12
13
"gitlab.com/Timfa/bluegit/internal/access"
14
"gitlab.com/Timfa/bluegit/internal/runner"
15
"gitlab.com/Timfa/bluegit/internal/store"
16
)
17
18
// RunnerView is één aparte runner in de beheer-UI.
19
type RunnerView struct {
20
ID int64
21
Name string
22
Online bool
23
LastSeen time.Time
24
Created time.Time
25
Version string
26
}
27
28
func runnerViews(rs []store.Runner) []RunnerView {
29
now := time.Now()
30
out := make([]RunnerView, 0, len(rs))
31
for _, r := range rs {
32
out = append(out, RunnerView{ID: r.ID, Name: r.Name, Online: r.Online(now), LastSeen: r.LastSeen, Created: r.CreatedAt, Version: r.Version})
33
}
34
return out
35
}
36
37
// Long-poll-duur voor runner-jobaanvragen.
38
var runnerPollWait = 25 * time.Second
39
40
func (s *Server) routeRunners(mux *http.ServeMux) {
41
mux.HandleFunc("POST /settings/runners/token", s.userRunnerToken)
42
mux.HandleFunc("POST /settings/runners/delete", s.userRunnerDelete)
43
mux.HandleFunc("POST /settings/runners/pref", s.userRunnerPref)
44
mux.HandleFunc("POST /groups/{path}/runners/token", s.groupRunnerToken)
45
mux.HandleFunc("POST /groups/{path}/runners/delete", s.groupRunnerDelete)
46
mux.HandleFunc("POST /groups/{path}/runners/options", s.groupRunnerOptions)
47
mux.HandleFunc("POST /groups/{path}/runners/optin", s.groupRunnerOptIn)
48
49
mux.HandleFunc("POST /api/v1/runners/register", s.runnerRegister)
50
mux.HandleFunc("POST /api/v1/runner/heartbeat", s.runnerHeartbeat)
51
mux.HandleFunc("POST /api/v1/runner/jobs/request", s.runnerRequest)
52
mux.HandleFunc("POST /api/v1/runner/jobs/{id}/log", s.runnerLog)
53
mux.HandleFunc("GET /api/v1/runner/jobs/{id}/inputs", s.runnerInputs)
54
mux.HandleFunc("PUT /api/v1/runner/jobs/{id}/artifacts", s.runnerArtifacts)
55
mux.HandleFunc("POST /api/v1/runner/jobs/{id}/finish", s.runnerFinish)
56
}
57
58
// ---- beheer-UI ----
59
60
func (s *Server) setRunnerOnce(w http.ResponseWriter, path, plain string) {
61
secure := strings.HasPrefix(s.Cfg.BaseURL, "https://")
62
http.SetCookie(w, &http.Cookie{Name: "bluegit_runner_once", Value: plain, Path: path, MaxAge: 120, HttpOnly: true, Secure: secure, SameSite: http.SameSiteLaxMode})
63
}
64
65
func (s *Server) takeRunnerOnce(w http.ResponseWriter, r *http.Request, path string) string {
66
v := cookieVal(r, "bluegit_runner_once")
67
if v != "" {
68
http.SetCookie(w, &http.Cookie{Name: "bluegit_runner_once", Value: "", Path: path, MaxAge: -1})
69
}
70
return v
71
}
72
73
func (s *Server) fillUserRunners(w http.ResponseWriter, r *http.Request, v *View, u store.User) {
74
rs, _ := s.Store.ListRunners(r.Context(), "user", u.ID)
75
v.RunnerList = runnerViews(rs)
76
v.RunnerPref = s.Store.UserRunnerPref(r.Context(), u.ID)
77
v.NewRunnerToken = s.takeRunnerOnce(w, r, "/settings")
78
}
79
80
func (s *Server) userRunnerToken(w http.ResponseWriter, r *http.Request) {
81
u, ok := s.loginRequired(w, r)
82
if !ok || !s.csrfOK(w, r) {
83
return
84
}
85
plain, err := s.Store.CreateRunnerRegToken(r.Context(), "user", u.ID, u.ID)
86
if err != nil {
87
s.redirect(w, r, "/settings#runners", "", s.trErr(r, err))
88
return
89
}
90
s.setRunnerOnce(w, "/settings", plain)
91
s.redirect(w, r, "/settings#runners", s.tr(r, "runner.tokenMade"), "")
92
}
93
94
func (s *Server) userRunnerDelete(w http.ResponseWriter, r *http.Request) {
95
u, ok := s.loginRequired(w, r)
96
if !ok || !s.csrfOK(w, r) {
97
return
98
}
99
id, _ := strconv.ParseInt(r.FormValue("id"), 10, 64)
100
if err := s.Store.RevokeRunner(r.Context(), "user", u.ID, id); err != nil {
101
s.redirect(w, r, "/settings#runners", "", s.tr(r, "runner.missing"))
102
return
103
}
104
s.redirect(w, r, "/settings#runners", s.tr(r, "runner.revoked"), "")
105
}
106
107
func (s *Server) userRunnerPref(w http.ResponseWriter, r *http.Request) {
108
u, ok := s.loginRequired(w, r)
109
if !ok || !s.csrfOK(w, r) {
110
return
111
}
112
_ = s.Store.SetUserRunnerPref(r.Context(), u.ID, r.FormValue("pref"))
113
s.redirect(w, r, "/settings#runners", s.tr(r, "flash.saved"), "")
114
}
115
116
func (s *Server) fillGroupRunners(w http.ResponseWriter, r *http.Request, v *View, g store.Group) {
117
rs, _ := s.Store.ListRunners(r.Context(), "group", g.ID)
118
v.RunnerList = runnerViews(rs)
119
v.BuiltinFirst = s.Store.GroupBuiltinFirst(r.Context(), g.ID)
120
ids, _ := s.Store.RunnerOptIns(r.Context(), g.ID)
121
for _, id := range ids {
122
if v.User != nil && id == v.User.ID {
123
v.RunnerOptIn = true
124
}
125
if u, err := s.Store.UserByID(r.Context(), id); err == nil {
126
v.OptInNames = append(v.OptInNames, u.Username)
127
}
128
}
129
v.NewRunnerToken = s.takeRunnerOnce(w, r, "/groups/"+g.Path)
130
}
131
132
func (s *Server) groupRunnerToken(w http.ResponseWriter, r *http.Request) {
133
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)
134
if !ok || !s.csrfOK(w, r) {
135
return
136
}
137
back := "/groups/" + g.Path + "?tab=settings#runners"
138
plain, err := s.Store.CreateRunnerRegToken(r.Context(), "group", g.ID, s.who(r).User.ID)
139
if err != nil {
140
s.redirect(w, r, back, "", s.trErr(r, err))
141
return
142
}
143
s.setRunnerOnce(w, "/groups/"+g.Path, plain)
144
s.redirect(w, r, back, s.tr(r, "runner.tokenMade"), "")
145
}
146
147
func (s *Server) groupRunnerDelete(w http.ResponseWriter, r *http.Request) {
148
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)
149
if !ok || !s.csrfOK(w, r) {
150
return
151
}
152
back := "/groups/" + g.Path + "?tab=settings#runners"
153
id, _ := strconv.ParseInt(r.FormValue("id"), 10, 64)
154
if err := s.Store.RevokeRunner(r.Context(), "group", g.ID, id); err != nil {
155
s.redirect(w, r, back, "", s.tr(r, "runner.missing"))
156
return
157
}
158
s.redirect(w, r, back, s.tr(r, "runner.revoked"), "")
159
}
160
161
func (s *Server) groupRunnerOptions(w http.ResponseWriter, r *http.Request) {
162
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)
163
if !ok || !s.csrfOK(w, r) {
164
return
165
}
166
_ = s.Store.SetGroupBuiltinFirst(r.Context(), g.ID, r.FormValue("builtin_first") == "1")
167
s.redirect(w, r, "/groups/"+g.Path+"?tab=settings#runners", s.tr(r, "flash.saved"), "")
168
}
169
170
// groupRunnerOptIn: een maintainer staat zijn eigen account-runners toe
171
// voor de groep (of trekt dat in). Alleen voor zichzelf.
172
func (s *Server) groupRunnerOptIn(w http.ResponseWriter, r *http.Request) {
173
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)
174
if !ok || !s.csrfOK(w, r) {
175
return
176
}
177
_ = s.Store.SetRunnerOptIn(r.Context(), g.ID, s.who(r).User.ID, r.FormValue("on") == "1")
178
s.redirect(w, r, "/groups/"+g.Path+"?tab=settings#runners", s.tr(r, "flash.saved"), "")
179
}
180
181
// ---- runner-protocol ----
182
183
func runnerJSONErr(w http.ResponseWriter, code int, msg string) {
184
writeJSON(w, code, map[string]any{"message": msg})
185
}
186
187
func (s *Server) runnerAuth(w http.ResponseWriter, r *http.Request) (store.Runner, bool) {
188
h := r.Header.Get("Authorization")
189
tok := ""
190
if strings.HasPrefix(strings.ToLower(h), "bearer ") {
191
tok = strings.TrimSpace(h[7:])
192
}
193
rn, err := s.Store.RunnerByToken(r.Context(), tok)
194
if err != nil {
195
runnerJSONErr(w, http.StatusUnauthorized, "invalid runner token")
196
return store.Runner{}, false
197
}
198
if s.Runner == nil || s.StoragePath() == "" {
199
runnerJSONErr(w, http.StatusServiceUnavailable, "CI is disabled on this server")
200
return store.Runner{}, false
201
}
202
_ = s.Store.TouchRunner(r.Context(), rn.ID, r.Header.Get("X-Runner-Version"))
203
return rn, true
204
}
205
206
func (s *Server) runnerJob(w http.ResponseWriter, r *http.Request, rn store.Runner) (store.Job, bool) {
207
id, _ := strconv.ParseInt(r.PathValue("id"), 10, 64)
208
job, err := s.Store.JobForRunner(r.Context(), rn.ID, id)
209
if err != nil {
210
runnerJSONErr(w, http.StatusNotFound, "job not found")
211
return store.Job{}, false
212
}
213
return job, true
214
}
215
216
func (s *Server) runnerRegister(w http.ResponseWriter, r *http.Request) {
217
var body struct {
218
Token string `json:"token"`
219
Name string `json:"name"`
220
Version string `json:"version"`
221
}
222
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&body); err != nil {
223
runnerJSONErr(w, http.StatusBadRequest, "invalid JSON")
224
return
225
}
226
rn, plain, err := s.Store.RegisterRunner(r.Context(), strings.TrimSpace(body.Token), body.Name, body.Version)
227
if err != nil {
228
if errors.Is(err, store.ErrRegToken) {
229
runnerJSONErr(w, http.StatusForbidden, err.Error())
230
return
231
}
232
runnerJSONErr(w, http.StatusInternalServerError, "registration failed")
233
return
234
}
235
writeJSON(w, http.StatusCreated, map[string]any{"id": rn.ID, "name": rn.Name, "token": plain})
236
}
237
238
func (s *Server) runnerHeartbeat(w http.ResponseWriter, r *http.Request) {
239
rn, ok := s.runnerAuth(w, r)
240
if !ok {
241
return
242
}
243
writeJSON(w, http.StatusOK, map[string]any{"id": rn.ID, "name": rn.Name})
244
}
245
246
func (s *Server) runnerRequest(w http.ResponseWriter, r *http.Request) {
247
rn, ok := s.runnerAuth(w, r)
248
if !ok {
249
return
250
}
251
deadline := time.Now().Add(runnerPollWait)
252
for {
253
pl, err := s.Runner.Claim(r.Context(), rn)
254
if err != nil {
255
runnerJSONErr(w, http.StatusInternalServerError, "claim failed")
256
return
257
}
258
if pl != nil {
259
writeJSON(w, http.StatusOK, pl)
260
return
261
}
262
if time.Now().After(deadline) {
263
w.WriteHeader(http.StatusNoContent)
264
return
265
}
266
select {
267
case <-r.Context().Done():
268
return
269
case <-time.After(time.Second):
270
}
271
_ = s.Store.TouchRunner(r.Context(), rn.ID, "")
272
}
273
}
274
275
func (s *Server) runnerLog(w http.ResponseWriter, r *http.Request) {
276
rn, ok := s.runnerAuth(w, r)
277
if !ok {
278
return
279
}
280
job, ok := s.runnerJob(w, r, rn)
281
if !ok {
282
return
283
}
284
chunk, _ := io.ReadAll(io.LimitReader(r.Body, 4<<20))
285
cancel, err := s.Runner.RemoteLog(r.Context(), job, string(chunk))
286
if err != nil {
287
runnerJSONErr(w, http.StatusInternalServerError, "log failed")
288
return
289
}
290
writeJSON(w, http.StatusOK, map[string]any{"cancel": cancel})
291
}
292
293
func (s *Server) runnerInputs(w http.ResponseWriter, r *http.Request) {
294
rn, ok := s.runnerAuth(w, r)
295
if !ok {
296
return
297
}
298
job, ok := s.runnerJob(w, r, rn)
299
if !ok || job.Status != "running" {
300
if ok {
301
runnerJSONErr(w, http.StatusConflict, "job is not running")
302
}
303
return
304
}
305
dir, err := os.MkdirTemp("", "bluegit-inputs-")
306
if err != nil {
307
runnerJSONErr(w, http.StatusInternalServerError, "temp dir")
308
return
309
}
310
defer os.RemoveAll(dir)
311
if err := s.Runner.RemoteInputs(r.Context(), job, dir); err != nil {
312
runnerJSONErr(w, http.StatusInternalServerError, "inputs failed")
313
return
314
}
315
w.Header().Set("Content-Type", "application/gzip")
316
_ = runner.WriteTar(w, dir, nil)
317
}
318
319
func (s *Server) runnerArtifacts(w http.ResponseWriter, r *http.Request) {
320
rn, ok := s.runnerAuth(w, r)
321
if !ok {
322
return
323
}
324
job, ok := s.runnerJob(w, r, rn)
325
if !ok || job.Status != "running" {
326
if ok {
327
runnerJSONErr(w, http.StatusConflict, "job is not running")
328
}
329
return
330
}
331
dir, err := os.MkdirTemp("", "bluegit-artifacts-")
332
if err != nil {
333
runnerJSONErr(w, http.StatusInternalServerError, "temp dir")
334
return
335
}
336
defer os.RemoveAll(dir)
337
paths := job.ArtifactPaths
338
if paths == nil {
339
paths = []string{}
340
}
341
if err := runner.ReadTar(r.Body, dir, paths, 4<<30); err != nil {
342
runnerJSONErr(w, http.StatusBadRequest, "bad archive: "+err.Error())
343
return
344
}
345
if err := s.Runner.RemoteArtifacts(r.Context(), job, dir); err != nil {
346
runnerJSONErr(w, http.StatusBadRequest, err.Error())
347
return
348
}
349
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
350
}
351
352
func (s *Server) runnerFinish(w http.ResponseWriter, r *http.Request) {
353
rn, ok := s.runnerAuth(w, r)
354
if !ok {
355
return
356
}
357
job, ok := s.runnerJob(w, r, rn)
358
if !ok {
359
return
360
}
361
var body struct {
362
Status string `json:"status"`
363
Message string `json:"message"`
364
}
365
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil {
366
runnerJSONErr(w, http.StatusBadRequest, "invalid JSON")
367
return
368
}
369
if err := s.Runner.RemoteFinish(r.Context(), rn, job.ID, body.Status, body.Message); err != nil {
370
runnerJSONErr(w, http.StatusInternalServerError, "finish failed")
371
return
372
}
373
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
374
}
@@ -0,0 +1,374 @@1
package server2
3
import (4
"encoding/json"5
"errors"6
"io"7
"net/http"8
"os"9
"strconv"10
"strings"11
"time"12
13
"gitlab.com/Timfa/bluegit/internal/access"14
"gitlab.com/Timfa/bluegit/internal/runner"15
"gitlab.com/Timfa/bluegit/internal/store"16
)17
18
// RunnerView is één aparte runner in de beheer-UI.19
type RunnerView struct {20
ID int6421
Name string22
Online bool23
LastSeen time.Time24
Created time.Time25
Version string26
}27
28
func runnerViews(rs []store.Runner) []RunnerView {29
now := time.Now()30
out := make([]RunnerView, 0, len(rs))31
for _, r := range rs {32
out = append(out, RunnerView{ID: r.ID, Name: r.Name, Online: r.Online(now), LastSeen: r.LastSeen, Created: r.CreatedAt, Version: r.Version})33
}34
return out35
}36
37
// Long-poll-duur voor runner-jobaanvragen.38
var runnerPollWait = 25 * time.Second39
40
func (s *Server) routeRunners(mux *http.ServeMux) {41
mux.HandleFunc("POST /settings/runners/token", s.userRunnerToken)42
mux.HandleFunc("POST /settings/runners/delete", s.userRunnerDelete)43
mux.HandleFunc("POST /settings/runners/pref", s.userRunnerPref)44
mux.HandleFunc("POST /groups/{path}/runners/token", s.groupRunnerToken)45
mux.HandleFunc("POST /groups/{path}/runners/delete", s.groupRunnerDelete)46
mux.HandleFunc("POST /groups/{path}/runners/options", s.groupRunnerOptions)47
mux.HandleFunc("POST /groups/{path}/runners/optin", s.groupRunnerOptIn)48
49
mux.HandleFunc("POST /api/v1/runners/register", s.runnerRegister)50
mux.HandleFunc("POST /api/v1/runner/heartbeat", s.runnerHeartbeat)51
mux.HandleFunc("POST /api/v1/runner/jobs/request", s.runnerRequest)52
mux.HandleFunc("POST /api/v1/runner/jobs/{id}/log", s.runnerLog)53
mux.HandleFunc("GET /api/v1/runner/jobs/{id}/inputs", s.runnerInputs)54
mux.HandleFunc("PUT /api/v1/runner/jobs/{id}/artifacts", s.runnerArtifacts)55
mux.HandleFunc("POST /api/v1/runner/jobs/{id}/finish", s.runnerFinish)56
}57
58
// ---- beheer-UI ----59
60
func (s *Server) setRunnerOnce(w http.ResponseWriter, path, plain string) {61
secure := strings.HasPrefix(s.Cfg.BaseURL, "https://")62
http.SetCookie(w, &http.Cookie{Name: "bluegit_runner_once", Value: plain, Path: path, MaxAge: 120, HttpOnly: true, Secure: secure, SameSite: http.SameSiteLaxMode})63
}64
65
func (s *Server) takeRunnerOnce(w http.ResponseWriter, r *http.Request, path string) string {66
v := cookieVal(r, "bluegit_runner_once")67
if v != "" {68
http.SetCookie(w, &http.Cookie{Name: "bluegit_runner_once", Value: "", Path: path, MaxAge: -1})69
}70
return v71
}72
73
func (s *Server) fillUserRunners(w http.ResponseWriter, r *http.Request, v *View, u store.User) {74
rs, _ := s.Store.ListRunners(r.Context(), "user", u.ID)75
v.RunnerList = runnerViews(rs)76
v.RunnerPref = s.Store.UserRunnerPref(r.Context(), u.ID)77
v.NewRunnerToken = s.takeRunnerOnce(w, r, "/settings")78
}79
80
func (s *Server) userRunnerToken(w http.ResponseWriter, r *http.Request) {81
u, ok := s.loginRequired(w, r)82
if !ok || !s.csrfOK(w, r) {83
return84
}85
plain, err := s.Store.CreateRunnerRegToken(r.Context(), "user", u.ID, u.ID)86
if err != nil {87
s.redirect(w, r, "/settings#runners", "", s.trErr(r, err))88
return89
}90
s.setRunnerOnce(w, "/settings", plain)91
s.redirect(w, r, "/settings#runners", s.tr(r, "runner.tokenMade"), "")92
}93
94
func (s *Server) userRunnerDelete(w http.ResponseWriter, r *http.Request) {95
u, ok := s.loginRequired(w, r)96
if !ok || !s.csrfOK(w, r) {97
return98
}99
id, _ := strconv.ParseInt(r.FormValue("id"), 10, 64)100
if err := s.Store.RevokeRunner(r.Context(), "user", u.ID, id); err != nil {101
s.redirect(w, r, "/settings#runners", "", s.tr(r, "runner.missing"))102
return103
}104
s.redirect(w, r, "/settings#runners", s.tr(r, "runner.revoked"), "")105
}106
107
func (s *Server) userRunnerPref(w http.ResponseWriter, r *http.Request) {108
u, ok := s.loginRequired(w, r)109
if !ok || !s.csrfOK(w, r) {110
return111
}112
_ = s.Store.SetUserRunnerPref(r.Context(), u.ID, r.FormValue("pref"))113
s.redirect(w, r, "/settings#runners", s.tr(r, "flash.saved"), "")114
}115
116
func (s *Server) fillGroupRunners(w http.ResponseWriter, r *http.Request, v *View, g store.Group) {117
rs, _ := s.Store.ListRunners(r.Context(), "group", g.ID)118
v.RunnerList = runnerViews(rs)119
v.BuiltinFirst = s.Store.GroupBuiltinFirst(r.Context(), g.ID)120
ids, _ := s.Store.RunnerOptIns(r.Context(), g.ID)121
for _, id := range ids {122
if v.User != nil && id == v.User.ID {123
v.RunnerOptIn = true124
}125
if u, err := s.Store.UserByID(r.Context(), id); err == nil {126
v.OptInNames = append(v.OptInNames, u.Username)127
}128
}129
v.NewRunnerToken = s.takeRunnerOnce(w, r, "/groups/"+g.Path)130
}131
132
func (s *Server) groupRunnerToken(w http.ResponseWriter, r *http.Request) {133
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)134
if !ok || !s.csrfOK(w, r) {135
return136
}137
back := "/groups/" + g.Path + "?tab=settings#runners"138
plain, err := s.Store.CreateRunnerRegToken(r.Context(), "group", g.ID, s.who(r).User.ID)139
if err != nil {140
s.redirect(w, r, back, "", s.trErr(r, err))141
return142
}143
s.setRunnerOnce(w, "/groups/"+g.Path, plain)144
s.redirect(w, r, back, s.tr(r, "runner.tokenMade"), "")145
}146
147
func (s *Server) groupRunnerDelete(w http.ResponseWriter, r *http.Request) {148
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)149
if !ok || !s.csrfOK(w, r) {150
return151
}152
back := "/groups/" + g.Path + "?tab=settings#runners"153
id, _ := strconv.ParseInt(r.FormValue("id"), 10, 64)154
if err := s.Store.RevokeRunner(r.Context(), "group", g.ID, id); err != nil {155
s.redirect(w, r, back, "", s.tr(r, "runner.missing"))156
return157
}158
s.redirect(w, r, back, s.tr(r, "runner.revoked"), "")159
}160
161
func (s *Server) groupRunnerOptions(w http.ResponseWriter, r *http.Request) {162
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)163
if !ok || !s.csrfOK(w, r) {164
return165
}166
_ = s.Store.SetGroupBuiltinFirst(r.Context(), g.ID, r.FormValue("builtin_first") == "1")167
s.redirect(w, r, "/groups/"+g.Path+"?tab=settings#runners", s.tr(r, "flash.saved"), "")168
}169
170
// groupRunnerOptIn: een maintainer staat zijn eigen account-runners toe171
// voor de groep (of trekt dat in). Alleen voor zichzelf.172
func (s *Server) groupRunnerOptIn(w http.ResponseWriter, r *http.Request) {173
g, _, ok := s.guardGroup(w, r, access.RoleMaintainer)174
if !ok || !s.csrfOK(w, r) {175
return176
}177
_ = s.Store.SetRunnerOptIn(r.Context(), g.ID, s.who(r).User.ID, r.FormValue("on") == "1")178
s.redirect(w, r, "/groups/"+g.Path+"?tab=settings#runners", s.tr(r, "flash.saved"), "")179
}180
181
// ---- runner-protocol ----182
183
func runnerJSONErr(w http.ResponseWriter, code int, msg string) {184
writeJSON(w, code, map[string]any{"message": msg})185
}186
187
func (s *Server) runnerAuth(w http.ResponseWriter, r *http.Request) (store.Runner, bool) {188
h := r.Header.Get("Authorization")189
tok := ""190
if strings.HasPrefix(strings.ToLower(h), "bearer ") {191
tok = strings.TrimSpace(h[7:])192
}193
rn, err := s.Store.RunnerByToken(r.Context(), tok)194
if err != nil {195
runnerJSONErr(w, http.StatusUnauthorized, "invalid runner token")196
return store.Runner{}, false197
}198
if s.Runner == nil || s.StoragePath() == "" {199
runnerJSONErr(w, http.StatusServiceUnavailable, "CI is disabled on this server")200
return store.Runner{}, false201
}202
_ = s.Store.TouchRunner(r.Context(), rn.ID, r.Header.Get("X-Runner-Version"))203
return rn, true204
}205
206
func (s *Server) runnerJob(w http.ResponseWriter, r *http.Request, rn store.Runner) (store.Job, bool) {207
id, _ := strconv.ParseInt(r.PathValue("id"), 10, 64)208
job, err := s.Store.JobForRunner(r.Context(), rn.ID, id)209
if err != nil {210
runnerJSONErr(w, http.StatusNotFound, "job not found")211
return store.Job{}, false212
}213
return job, true214
}215
216
func (s *Server) runnerRegister(w http.ResponseWriter, r *http.Request) {217
var body struct {218
Token string `json:"token"`219
Name string `json:"name"`220
Version string `json:"version"`221
}222
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&body); err != nil {223
runnerJSONErr(w, http.StatusBadRequest, "invalid JSON")224
return225
}226
rn, plain, err := s.Store.RegisterRunner(r.Context(), strings.TrimSpace(body.Token), body.Name, body.Version)227
if err != nil {228
if errors.Is(err, store.ErrRegToken) {229
runnerJSONErr(w, http.StatusForbidden, err.Error())230
return231
}232
runnerJSONErr(w, http.StatusInternalServerError, "registration failed")233
return234
}235
writeJSON(w, http.StatusCreated, map[string]any{"id": rn.ID, "name": rn.Name, "token": plain})236
}237
238
func (s *Server) runnerHeartbeat(w http.ResponseWriter, r *http.Request) {239
rn, ok := s.runnerAuth(w, r)240
if !ok {241
return242
}243
writeJSON(w, http.StatusOK, map[string]any{"id": rn.ID, "name": rn.Name})244
}245
246
func (s *Server) runnerRequest(w http.ResponseWriter, r *http.Request) {247
rn, ok := s.runnerAuth(w, r)248
if !ok {249
return250
}251
deadline := time.Now().Add(runnerPollWait)252
for {253
pl, err := s.Runner.Claim(r.Context(), rn)254
if err != nil {255
runnerJSONErr(w, http.StatusInternalServerError, "claim failed")256
return257
}258
if pl != nil {259
writeJSON(w, http.StatusOK, pl)260
return261
}262
if time.Now().After(deadline) {263
w.WriteHeader(http.StatusNoContent)264
return265
}266
select {267
case <-r.Context().Done():268
return269
case <-time.After(time.Second):270
}271
_ = s.Store.TouchRunner(r.Context(), rn.ID, "")272
}273
}274
275
func (s *Server) runnerLog(w http.ResponseWriter, r *http.Request) {276
rn, ok := s.runnerAuth(w, r)277
if !ok {278
return279
}280
job, ok := s.runnerJob(w, r, rn)281
if !ok {282
return283
}284
chunk, _ := io.ReadAll(io.LimitReader(r.Body, 4<<20))285
cancel, err := s.Runner.RemoteLog(r.Context(), job, string(chunk))286
if err != nil {287
runnerJSONErr(w, http.StatusInternalServerError, "log failed")288
return289
}290
writeJSON(w, http.StatusOK, map[string]any{"cancel": cancel})291
}292
293
func (s *Server) runnerInputs(w http.ResponseWriter, r *http.Request) {294
rn, ok := s.runnerAuth(w, r)295
if !ok {296
return297
}298
job, ok := s.runnerJob(w, r, rn)299
if !ok || job.Status != "running" {300
if ok {301
runnerJSONErr(w, http.StatusConflict, "job is not running")302
}303
return304
}305
dir, err := os.MkdirTemp("", "bluegit-inputs-")306
if err != nil {307
runnerJSONErr(w, http.StatusInternalServerError, "temp dir")308
return309
}310
defer os.RemoveAll(dir)311
if err := s.Runner.RemoteInputs(r.Context(), job, dir); err != nil {312
runnerJSONErr(w, http.StatusInternalServerError, "inputs failed")313
return314
}315
w.Header().Set("Content-Type", "application/gzip")316
_ = runner.WriteTar(w, dir, nil)317
}318
319
func (s *Server) runnerArtifacts(w http.ResponseWriter, r *http.Request) {320
rn, ok := s.runnerAuth(w, r)321
if !ok {322
return323
}324
job, ok := s.runnerJob(w, r, rn)325
if !ok || job.Status != "running" {326
if ok {327
runnerJSONErr(w, http.StatusConflict, "job is not running")328
}329
return330
}331
dir, err := os.MkdirTemp("", "bluegit-artifacts-")332
if err != nil {333
runnerJSONErr(w, http.StatusInternalServerError, "temp dir")334
return335
}336
defer os.RemoveAll(dir)337
paths := job.ArtifactPaths338
if paths == nil {339
paths = []string{}340
}341
if err := runner.ReadTar(r.Body, dir, paths, 4<<30); err != nil {342
runnerJSONErr(w, http.StatusBadRequest, "bad archive: "+err.Error())343
return344
}345
if err := s.Runner.RemoteArtifacts(r.Context(), job, dir); err != nil {346
runnerJSONErr(w, http.StatusBadRequest, err.Error())347
return348
}349
writeJSON(w, http.StatusOK, map[string]any{"ok": true})350
}351
352
func (s *Server) runnerFinish(w http.ResponseWriter, r *http.Request) {353
rn, ok := s.runnerAuth(w, r)354
if !ok {355
return356
}357
job, ok := s.runnerJob(w, r, rn)358
if !ok {359
return360
}361
var body struct {362
Status string `json:"status"`363
Message string `json:"message"`364
}365
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil {366
runnerJSONErr(w, http.StatusBadRequest, "invalid JSON")367
return368
}369
if err := s.Runner.RemoteFinish(r.Context(), rn, job.ID, body.Status, body.Message); err != nil {370
runnerJSONErr(w, http.StatusInternalServerError, "finish failed")371
return372
}373
writeJSON(w, http.StatusOK, map[string]any{"ok": true})374
}@@ -107,6 +107,7 @@ func (s *Server) Handler() http.Handler {
107
107
mux.HandleFunc("POST /setup", s.setupSave)
108
108
mux.HandleFunc("POST /setup/cache", s.setupCacheSave)
109
109
mux.HandleFunc("POST /setup/cache/clear", s.setupCacheClear)
110
s.routeRunners(mux)
110
111
mux.HandleFunc("GET /settings", s.account)
111
112
mux.HandleFunc("POST /settings/language", s.saveLanguage)
112
113
mux.HandleFunc("POST /settings/profile", s.saveProfile)
@@ -527,6 +528,12 @@ func randomState() (string, error) {
527
528
528
529
// View is de gegevensset voor elk scherm.
529
530
type View struct {
531
RunnerList []RunnerView
532
RunnerPref string
533
NewRunnerToken string
534
BuiltinFirst bool
535
RunnerOptIn bool
536
OptInNames []string
530
537
Title string
531
538
BaseURL string
532
539
RunnerName string
@@ -107,6 +107,7 @@ func (s *Server) Handler() http.Handler {107
mux.HandleFunc("POST /setup", s.setupSave)107
mux.HandleFunc("POST /setup", s.setupSave)108
mux.HandleFunc("POST /setup/cache", s.setupCacheSave)108
mux.HandleFunc("POST /setup/cache", s.setupCacheSave)109
mux.HandleFunc("POST /setup/cache/clear", s.setupCacheClear)109
mux.HandleFunc("POST /setup/cache/clear", s.setupCacheClear)110
s.routeRunners(mux)110
mux.HandleFunc("GET /settings", s.account)111
mux.HandleFunc("GET /settings", s.account)111
mux.HandleFunc("POST /settings/language", s.saveLanguage)112
mux.HandleFunc("POST /settings/language", s.saveLanguage)112
mux.HandleFunc("POST /settings/profile", s.saveProfile)113
mux.HandleFunc("POST /settings/profile", s.saveProfile)@@ -527,6 +528,12 @@ func randomState() (string, error) {527
528
528
// View is de gegevensset voor elk scherm.529
// View is de gegevensset voor elk scherm.529
type View struct {530
type View struct {531
RunnerList []RunnerView532
RunnerPref string533
NewRunnerToken string534
BuiltinFirst bool535
RunnerOptIn bool536
OptInNames []string530
Title string537
Title string531
BaseURL string538
BaseURL string532
RunnerName string539
RunnerName string@@ -631,6 +631,7 @@ func (s *Server) groupPage(w http.ResponseWriter, r *http.Request) {
631
631
v.Hooks = s.hookViews(r, "group", g.ID)
632
632
s.fillHookEdit(r, &v, "group", g.ID)
633
633
s.fillGroupApproval(r, &v, g)
634
s.fillGroupRunners(w, r, &v, g)
634
635
if v.User != nil && v.User.ID == g.OwnerID {
635
636
v.IsOwner = true
636
637
v.Candidates = groupCandidates(v.Members, g.OwnerID)
@@ -631,6 +631,7 @@ func (s *Server) groupPage(w http.ResponseWriter, r *http.Request) {631
v.Hooks = s.hookViews(r, "group", g.ID)631
v.Hooks = s.hookViews(r, "group", g.ID)632
s.fillHookEdit(r, &v, "group", g.ID)632
s.fillHookEdit(r, &v, "group", g.ID)633
s.fillGroupApproval(r, &v, g)633
s.fillGroupApproval(r, &v, g)634
s.fillGroupRunners(w, r, &v, g)634
if v.User != nil && v.User.ID == g.OwnerID {635
if v.User != nil && v.User.ID == g.OwnerID {635
v.IsOwner = true636
v.IsOwner = true636
v.Candidates = groupCandidates(v.Members, g.OwnerID)637
v.Candidates = groupCandidates(v.Members, g.OwnerID)@@ -447,7 +447,7 @@
447
447
</div>
448
448
</div>
449
449
<div class="console" data-log>
450
<p class="lognote">{{t "pipe.runningOn" $.RunnerName}}</p>
450
<p class="lognote">{{t "pipe.runningOn" (or .Runner $.RunnerName)}}</p>
451
451
<p class="lognote">{{t "pipe.utc"}}</p>
452
452
{{range .Blocks}}{{if .Section}}
453
453
<section class="logsec{{if .Open}} open{{end}}">
@@ -467,7 +467,7 @@
467
467
<dt>{{t "pipe.queued"}}</dt><dd>{{if .Queued}}{{.Queued}}{{else}}—{{end}}</dd>
468
468
<dt>{{t "pipe.timeout"}}</dt><dd>{{.Timeout}}</dd>
469
469
{{if .Environment}}<dt>{{t "pipe.environment"}}</dt><dd>{{.Environment}}{{if .EnvironmentLink}} <a href="{{.EnvironmentLink}}">{{.EnvironmentURL}}</a>{{else if .EnvironmentURL}} {{.EnvironmentURL}}{{end}}</dd>{{end}}
470
<dt>{{t "pipe.runner"}}</dt><dd>{{$.RunnerName}}</dd>
470
<dt>{{t "pipe.runner"}}</dt><dd>{{or .Runner $.RunnerName}}</dd>
471
471
<dt>{{t "pipe.source"}}</dt><dd>{{.Source}}</dd>
472
472
<dt>{{t "pipe.commit"}}</dt><dd><code>{{.Short}}</code> <button type="button" class="iconbtn" data-copy="{{.Pipe.CommitSHA}}" aria-label="{{t "action.copy"}}">{{t "action.copy"}}</button>{{if .Title}}<div>{{.Title}}</div>{{end}}</dd>
473
473
<dt>{{t "pipe.one"}}</dt><dd><a href="/{{$.Repo.FullPath}}/-/pipelines/{{.Pipe.ID}}">#{{.Pipe.ID}}</a> <span class="status {{.PipeStatus}}"><span class="mark"></span>{{.PipeLabel}}</span> <span class="muted">{{t "pipe.for"}} {{.Ref}}</span></dd>
@@ -919,6 +919,21 @@
919
919
<script src="/assets/transfer.js"></script>
920
920
{{else}}<p class="muted">{{t "own.needMember"}}</p>{{end}}
921
921
{{end}}
922
<h2 id="runners">{{t "runner.title"}}</h2>
923
<p class="muted">{{t "runner.groupLead"}}</p>
924
<form class="panel" method="post" action="/groups/{{.Group.Path}}/runners/options">
925
<input type="hidden" name="csrf" value="{{.CSRF}}">
926
<label><input type="checkbox" name="builtin_first" value="1" {{if .BuiltinFirst}}checked{{end}}> {{t "runner.builtinFirst"}}</label>
927
<button type="submit">{{t "action.save"}}</button>
928
</form>
929
<form class="panel" method="post" action="/groups/{{.Group.Path}}/runners/optin">
930
<input type="hidden" name="csrf" value="{{.CSRF}}">
931
<input type="hidden" name="on" value="{{if .RunnerOptIn}}0{{else}}1{{end}}">
932
<p>{{if .RunnerOptIn}}{{t "runner.optinOn"}}{{else}}{{t "runner.optinOff"}}{{end}}</p>
933
{{if .OptInNames}}<p class="muted">{{t "runner.optinWho"}} {{range $i, $n := .OptInNames}}{{if $i}}, {{end}}{{$n}}{{end}}</p>{{end}}
934
<button type="submit">{{if .RunnerOptIn}}{{t "runner.optinStop"}}{{else}}{{t "runner.optinStart"}}{{end}}</button>
935
</form>
936
{{template "runnerlist" (dict "Runners" .RunnerList "CSRF" .CSRF "Delete" (printf "/groups/%s/runners/delete" .Group.Path) "Token" .NewRunnerToken "New" (printf "/groups/%s/runners/token" .Group.Path) "Base" .BaseURL)}}
922
937
<h2>{{t "set.webhooks"}}</h2>
923
938
<p class="muted">{{t "group.secretsNote"}}</p>
924
939
{{template "webhookform" .}}
@@ -447,7 +447,7 @@447
</div>447
</div>448
</div>448
</div>449
<div class="console" data-log>449
<div class="console" data-log>450
<p class="lognote">{{t "pipe.runningOn" $.RunnerName}}</p>450
<p class="lognote">{{t "pipe.runningOn" (or .Runner $.RunnerName)}}</p>451
<p class="lognote">{{t "pipe.utc"}}</p>451
<p class="lognote">{{t "pipe.utc"}}</p>452
{{range .Blocks}}{{if .Section}}452
{{range .Blocks}}{{if .Section}}453
<section class="logsec{{if .Open}} open{{end}}">453
<section class="logsec{{if .Open}} open{{end}}">@@ -467,7 +467,7 @@467
<dt>{{t "pipe.queued"}}</dt><dd>{{if .Queued}}{{.Queued}}{{else}}—{{end}}</dd>467
<dt>{{t "pipe.queued"}}</dt><dd>{{if .Queued}}{{.Queued}}{{else}}—{{end}}</dd>468
<dt>{{t "pipe.timeout"}}</dt><dd>{{.Timeout}}</dd>468
<dt>{{t "pipe.timeout"}}</dt><dd>{{.Timeout}}</dd>469
{{if .Environment}}<dt>{{t "pipe.environment"}}</dt><dd>{{.Environment}}{{if .EnvironmentLink}} <a href="{{.EnvironmentLink}}">{{.EnvironmentURL}}</a>{{else if .EnvironmentURL}} {{.EnvironmentURL}}{{end}}</dd>{{end}}469
{{if .Environment}}<dt>{{t "pipe.environment"}}</dt><dd>{{.Environment}}{{if .EnvironmentLink}} <a href="{{.EnvironmentLink}}">{{.EnvironmentURL}}</a>{{else if .EnvironmentURL}} {{.EnvironmentURL}}{{end}}</dd>{{end}}470
<dt>{{t "pipe.runner"}}</dt><dd>{{$.RunnerName}}</dd>470
<dt>{{t "pipe.runner"}}</dt><dd>{{or .Runner $.RunnerName}}</dd>471
<dt>{{t "pipe.source"}}</dt><dd>{{.Source}}</dd>471
<dt>{{t "pipe.source"}}</dt><dd>{{.Source}}</dd>472
<dt>{{t "pipe.commit"}}</dt><dd><code>{{.Short}}</code> <button type="button" class="iconbtn" data-copy="{{.Pipe.CommitSHA}}" aria-label="{{t "action.copy"}}">{{t "action.copy"}}</button>{{if .Title}}<div>{{.Title}}</div>{{end}}</dd>472
<dt>{{t "pipe.commit"}}</dt><dd><code>{{.Short}}</code> <button type="button" class="iconbtn" data-copy="{{.Pipe.CommitSHA}}" aria-label="{{t "action.copy"}}">{{t "action.copy"}}</button>{{if .Title}}<div>{{.Title}}</div>{{end}}</dd>473
<dt>{{t "pipe.one"}}</dt><dd><a href="/{{$.Repo.FullPath}}/-/pipelines/{{.Pipe.ID}}">#{{.Pipe.ID}}</a> <span class="status {{.PipeStatus}}"><span class="mark"></span>{{.PipeLabel}}</span> <span class="muted">{{t "pipe.for"}} {{.Ref}}</span></dd>473
<dt>{{t "pipe.one"}}</dt><dd><a href="/{{$.Repo.FullPath}}/-/pipelines/{{.Pipe.ID}}">#{{.Pipe.ID}}</a> <span class="status {{.PipeStatus}}"><span class="mark"></span>{{.PipeLabel}}</span> <span class="muted">{{t "pipe.for"}} {{.Ref}}</span></dd>@@ -919,6 +919,21 @@919
<script src="/assets/transfer.js"></script>919
<script src="/assets/transfer.js"></script>920
{{else}}<p class="muted">{{t "own.needMember"}}</p>{{end}}920
{{else}}<p class="muted">{{t "own.needMember"}}</p>{{end}}921
{{end}}921
{{end}}922
<h2 id="runners">{{t "runner.title"}}</h2>923
<p class="muted">{{t "runner.groupLead"}}</p>924
<form class="panel" method="post" action="/groups/{{.Group.Path}}/runners/options">925
<input type="hidden" name="csrf" value="{{.CSRF}}">926
<label><input type="checkbox" name="builtin_first" value="1" {{if .BuiltinFirst}}checked{{end}}> {{t "runner.builtinFirst"}}</label>927
<button type="submit">{{t "action.save"}}</button>928
</form>929
<form class="panel" method="post" action="/groups/{{.Group.Path}}/runners/optin">930
<input type="hidden" name="csrf" value="{{.CSRF}}">931
<input type="hidden" name="on" value="{{if .RunnerOptIn}}0{{else}}1{{end}}">932
<p>{{if .RunnerOptIn}}{{t "runner.optinOn"}}{{else}}{{t "runner.optinOff"}}{{end}}</p>933
{{if .OptInNames}}<p class="muted">{{t "runner.optinWho"}} {{range $i, $n := .OptInNames}}{{if $i}}, {{end}}{{$n}}{{end}}</p>{{end}}934
<button type="submit">{{if .RunnerOptIn}}{{t "runner.optinStop"}}{{else}}{{t "runner.optinStart"}}{{end}}</button>935
</form>936
{{template "runnerlist" (dict "Runners" .RunnerList "CSRF" .CSRF "Delete" (printf "/groups/%s/runners/delete" .Group.Path) "Token" .NewRunnerToken "New" (printf "/groups/%s/runners/token" .Group.Path) "Base" .BaseURL)}}922
<h2>{{t "set.webhooks"}}</h2>937
<h2>{{t "set.webhooks"}}</h2>923
<p class="muted">{{t "group.secretsNote"}}</p>938
<p class="muted">{{t "group.secretsNote"}}</p>924
{{template "webhookform" .}}939
{{template "webhookform" .}}@@ -302,6 +302,17 @@
302
302
</div>
303
303
{{end}}
304
304
</section>
305
<section class="panel" id="runners">
306
<h2>{{t "runner.title"}}</h2>
307
<p class="muted">{{t "runner.userLead"}}</p>
308
<form method="post" action="/settings/runners/pref">
309
<input type="hidden" name="csrf" value="{{.CSRF}}">
310
<label><input type="radio" name="pref" value="builtin_first" {{if ne .RunnerPref "own_first"}}checked{{end}}> {{t "runner.prefBuiltin"}}</label>
311
<label><input type="radio" name="pref" value="own_first" {{if eq .RunnerPref "own_first"}}checked{{end}}> {{t "runner.prefOwn"}}</label>
312
<button type="submit">{{t "action.save"}}</button>
313
</form>
314
{{template "runnerlist" (dict "Runners" .RunnerList "CSRF" .CSRF "Delete" "/settings/runners/delete" "Token" .NewRunnerToken "New" "/settings/runners/token" "Base" .BaseURL)}}
315
</section>
305
316
<section class="panel">
306
317
<h2>{{t "acct.ssh"}}</h2>
307
318
<form method="post" action="/settings/ssh">
@@ -675,3 +686,33 @@
675
686
{{template "layout-end" .}}
676
687
{{end}}
677
688
689
690
{{define "runnerlist"}}
691
{{if .Token}}<p>{{t "runner.tokenOnce"}}</p><p class="code">{{.Token}}</p>
692
<pre class="code">sudo bluegit-runner register --url {{.Base}} --token {{.Token}}</pre>{{end}}
693
<form method="post" action="{{.New}}">
694
<input type="hidden" name="csrf" value="{{.CSRF}}">
695
<button type="submit">{{t "runner.new"}}</button>
696
</form>
697
{{if .Runners}}
698
<div class="hscroll">
699
<table class="acct">
700
<thead><tr><th>{{t "acct.col.name"}}</th><th>{{t "runner.status"}}</th><th>{{t "runner.seen"}}</th><th>{{t "runner.version"}}</th><th></th></tr></thead>
701
<tbody>
702
{{range .Runners}}<tr>
703
<td>{{.Name}}</td>
704
<td>{{if .Online}}<span class="pill">{{t "runner.online"}}</span>{{else}}<span class="muted">{{t "runner.offline"}}</span>{{end}}</td>
705
<td>{{date .LastSeen}}</td>
706
<td>{{.Version}}</td>
707
<td class="act">
708
<form method="post" action="{{$.Delete}}" data-confirm="{{t "runner.revokeAsk"}}">
709
<input type="hidden" name="csrf" value="{{$.CSRF}}"><input type="hidden" name="id" value="{{.ID}}">
710
<button class="danger" type="submit">{{t "acct.revoke"}}</button>
711
</form>
712
</td>
713
</tr>{{end}}
714
</tbody>
715
</table>
716
</div>
717
{{else}}<p class="muted">{{t "runner.none"}}</p>{{end}}
718
{{end}}
@@ -302,6 +302,17 @@302
</div>302
</div>303
{{end}}303
{{end}}304
</section>304
</section>305
<section class="panel" id="runners">306
<h2>{{t "runner.title"}}</h2>307
<p class="muted">{{t "runner.userLead"}}</p>308
<form method="post" action="/settings/runners/pref">309
<input type="hidden" name="csrf" value="{{.CSRF}}">310
<label><input type="radio" name="pref" value="builtin_first" {{if ne .RunnerPref "own_first"}}checked{{end}}> {{t "runner.prefBuiltin"}}</label>311
<label><input type="radio" name="pref" value="own_first" {{if eq .RunnerPref "own_first"}}checked{{end}}> {{t "runner.prefOwn"}}</label>312
<button type="submit">{{t "action.save"}}</button>313
</form>314
{{template "runnerlist" (dict "Runners" .RunnerList "CSRF" .CSRF "Delete" "/settings/runners/delete" "Token" .NewRunnerToken "New" "/settings/runners/token" "Base" .BaseURL)}}315
</section>305
<section class="panel">316
<section class="panel">306
<h2>{{t "acct.ssh"}}</h2>317
<h2>{{t "acct.ssh"}}</h2>307
<form method="post" action="/settings/ssh">318
<form method="post" action="/settings/ssh">@@ -675,3 +686,33 @@675
{{template "layout-end" .}}686
{{template "layout-end" .}}676
{{end}}687
{{end}}677
688
689
690
{{define "runnerlist"}}691
{{if .Token}}<p>{{t "runner.tokenOnce"}}</p><p class="code">{{.Token}}</p>692
<pre class="code">sudo bluegit-runner register --url {{.Base}} --token {{.Token}}</pre>{{end}}693
<form method="post" action="{{.New}}">694
<input type="hidden" name="csrf" value="{{.CSRF}}">695
<button type="submit">{{t "runner.new"}}</button>696
</form>697
{{if .Runners}}698
<div class="hscroll">699
<table class="acct">700
<thead><tr><th>{{t "acct.col.name"}}</th><th>{{t "runner.status"}}</th><th>{{t "runner.seen"}}</th><th>{{t "runner.version"}}</th><th></th></tr></thead>701
<tbody>702
{{range .Runners}}<tr>703
<td>{{.Name}}</td>704
<td>{{if .Online}}<span class="pill">{{t "runner.online"}}</span>{{else}}<span class="muted">{{t "runner.offline"}}</span>{{end}}</td>705
<td>{{date .LastSeen}}</td>706
<td>{{.Version}}</td>707
<td class="act">708
<form method="post" action="{{$.Delete}}" data-confirm="{{t "runner.revokeAsk"}}">709
<input type="hidden" name="csrf" value="{{$.CSRF}}"><input type="hidden" name="id" value="{{.ID}}">710
<button class="danger" type="submit">{{t "acct.revoke"}}</button>711
</form>712
</td>713
</tr>{{end}}714
</tbody>715
</table>716
</div>717
{{else}}<p class="muted">{{t "runner.none"}}</p>{{end}}718
{{end}}@@ -0,0 +1,352 @@
1
package store
2
3
import (
4
"context"
5
"database/sql"
6
"errors"
7
"strings"
8
"time"
9
)
10
11
// Runner is een aparte bluegit-runner op een eigen machine, gekoppeld aan
12
// een account (owner_kind user) of een groep (owner_kind group).
13
type Runner struct {
14
ID int64
15
OwnerKind string
16
OwnerID int64
17
Name string
18
CreatedAt time.Time
19
LastSeen time.Time
20
Version string
21
Revoked bool
22
}
23
24
// RunnerOnlineWindow: zo lang na de laatste heartbeat telt een runner als online.
25
const RunnerOnlineWindow = 90 * time.Second
26
27
// Online zegt of de runner recent contact had.
28
func (r Runner) Online(now time.Time) bool {
29
return !r.Revoked && !r.LastSeen.IsZero() && now.Sub(r.LastSeen) < RunnerOnlineWindow
30
}
31
32
// Runner-voorkeuren van een account.
33
const (
34
RunnerPrefOwnFirst = "own_first"
35
RunnerPrefBuiltinFirst = "builtin_first"
36
)
37
38
// ErrRegToken is een onbekend, gebruikt of verlopen registratietoken.
39
var ErrRegToken = errors.New("registration token invalid or expired")
40
41
// CreateRunnerRegToken maakt een eenmalig registratietoken (24 uur geldig).
42
func (s *Store) CreateRunnerRegToken(ctx context.Context, ownerKind string, ownerID, by int64) (string, error) {
43
plain, err := newToken("bgrr_")
44
if err != nil {
45
return "", err
46
}
47
exp := time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)
48
_, err = s.db.ExecContext(ctx, `INSERT INTO runner_reg_tokens (owner_kind, owner_id, token_hash, created_by, expires_at) VALUES (?, ?, ?, ?, ?)`,
49
ownerKind, ownerID, hashToken(plain), by, exp)
50
return plain, err
51
}
52
53
// RegisterRunner wisselt een registratietoken in voor een runner en zijn
54
// eigen runner-token (bgr_). Alleen de hash wordt bewaard.
55
func (s *Store) RegisterRunner(ctx context.Context, regToken, name, version string) (Runner, string, error) {
56
tx, err := s.db.BeginTx(ctx, nil)
57
if err != nil {
58
return Runner{}, "", err
59
}
60
defer func() { _ = tx.Rollback() }()
61
var id, ownerID, by int64
62
var kind, exp string
63
err = tx.QueryRowContext(ctx, `SELECT id, owner_kind, owner_id, created_by, expires_at FROM runner_reg_tokens WHERE token_hash = ? AND used = 0`, hashToken(regToken)).Scan(&id, &kind, &ownerID, &by, &exp)
64
if err != nil || parseTime(exp).Before(time.Now()) {
65
return Runner{}, "", ErrRegToken
66
}
67
if _, err := tx.ExecContext(ctx, `UPDATE runner_reg_tokens SET used = 1 WHERE id = ?`, id); err != nil {
68
return Runner{}, "", err
69
}
70
plain, err := newToken("bgr_")
71
if err != nil {
72
return Runner{}, "", err
73
}
74
name = strings.TrimSpace(name)
75
if name == "" {
76
name = "runner"
77
}
78
if len(name) > 80 {
79
name = name[:80]
80
}
81
at := now()
82
res, err := tx.ExecContext(ctx, `INSERT INTO runners (owner_kind, owner_id, name, token_hash, created_by, created_at, last_seen, version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
83
kind, ownerID, name, hashToken(plain), by, at, at, version)
84
if err != nil {
85
return Runner{}, "", err
86
}
87
rid, _ := res.LastInsertId()
88
if err := tx.Commit(); err != nil {
89
return Runner{}, "", err
90
}
91
return Runner{ID: rid, OwnerKind: kind, OwnerID: ownerID, Name: name, CreatedAt: parseTime(at), LastSeen: parseTime(at), Version: version}, plain, nil
92
}
93
94
const runnerCols = `id, owner_kind, owner_id, name, created_at, last_seen, version, revoked`
95
96
func scanRunner(sc interface{ Scan(...any) error }) (Runner, error) {
97
var r Runner
98
var created, seen string
99
var revoked int
100
if err := sc.Scan(&r.ID, &r.OwnerKind, &r.OwnerID, &r.Name, &created, &seen, &r.Version, &revoked); err != nil {
101
return Runner{}, err
102
}
103
r.CreatedAt, r.LastSeen, r.Revoked = parseTime(created), parseTime(seen), revoked == 1
104
return r, nil
105
}
106
107
// RunnerByToken geeft de runner bij een bgr_-token. Ingetrokken telt niet.
108
func (s *Store) RunnerByToken(ctx context.Context, token string) (Runner, error) {
109
if !strings.HasPrefix(token, "bgr_") {
110
return Runner{}, errNotFound
111
}
112
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE token_hash = ? AND revoked = 0`, hashToken(token)))
113
if errors.Is(err, sql.ErrNoRows) {
114
return Runner{}, errNotFound
115
}
116
return r, err
117
}
118
119
// RunnerByID geeft één runner.
120
func (s *Store) RunnerByID(ctx context.Context, id int64) (Runner, error) {
121
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE id = ?`, id))
122
if errors.Is(err, sql.ErrNoRows) {
123
return Runner{}, errNotFound
124
}
125
return r, err
126
}
127
128
// TouchRunner is de heartbeat.
129
func (s *Store) TouchRunner(ctx context.Context, id int64, version string) error {
130
_, err := s.db.ExecContext(ctx, `UPDATE runners SET last_seen = ?, version = CASE WHEN ? = '' THEN version ELSE ? END WHERE id = ?`, now(), version, version, id)
131
return err
132
}
133
134
// ListRunners geeft de niet-ingetrokken runners van een eigenaar.
135
func (s *Store) ListRunners(ctx context.Context, ownerKind string, ownerID int64) ([]Runner, error) {
136
rows, err := s.db.QueryContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE owner_kind = ? AND owner_id = ? AND revoked = 0 ORDER BY id`, ownerKind, ownerID)
137
if err != nil {
138
return nil, err
139
}
140
defer rows.Close()
141
var out []Runner
142
for rows.Next() {
143
r, err := scanRunner(rows)
144
if err != nil {
145
return nil, err
146
}
147
out = append(out, r)
148
}
149
return out, rows.Err()
150
}
151
152
// RevokeRunner trekt een runner in; zijn token werkt daarna niet meer.
153
func (s *Store) RevokeRunner(ctx context.Context, ownerKind string, ownerID, id int64) error {
154
res, err := s.db.ExecContext(ctx, `UPDATE runners SET revoked = 1 WHERE id = ? AND owner_kind = ? AND owner_id = ?`, id, ownerKind, ownerID)
155
if err != nil {
156
return err
157
}
158
if n, _ := res.RowsAffected(); n == 0 {
159
return errNotFound
160
}
161
return nil
162
}
163
164
// UserRunnerPref is own_first of builtin_first.
165
func (s *Store) UserRunnerPref(ctx context.Context, userID int64) string {
166
var p string
167
if err := s.db.QueryRowContext(ctx, `SELECT runner_pref FROM users WHERE id = ?`, userID).Scan(&p); err != nil || p != RunnerPrefOwnFirst {
168
return RunnerPrefBuiltinFirst
169
}
170
return p
171
}
172
173
// SetUserRunnerPref bewaart de voorkeur.
174
func (s *Store) SetUserRunnerPref(ctx context.Context, userID int64, pref string) error {
175
if pref != RunnerPrefOwnFirst {
176
pref = RunnerPrefBuiltinFirst
177
}
178
_, err := s.db.ExecContext(ctx, `UPDATE users SET runner_pref = ? WHERE id = ?`, pref, userID)
179
return err
180
}
181
182
// GroupBuiltinFirst: "Probeer BlueGit runner eerst".
183
func (s *Store) GroupBuiltinFirst(ctx context.Context, groupID int64) bool {
184
var v int
185
_ = s.db.QueryRowContext(ctx, `SELECT builtin_first FROM groups WHERE id = ?`, groupID).Scan(&v)
186
return v == 1
187
}
188
189
// SetGroupBuiltinFirst zet de groepsinstelling.
190
func (s *Store) SetGroupBuiltinFirst(ctx context.Context, groupID int64, on bool) error {
191
v := 0
192
if on {
193
v = 1
194
}
195
_, err := s.db.ExecContext(ctx, `UPDATE groups SET builtin_first = ? WHERE id = ?`, v, groupID)
196
return err
197
}
198
199
// SetRunnerOptIn: maintainer userID staat zijn account-runners toe voor de groep.
200
func (s *Store) SetRunnerOptIn(ctx context.Context, groupID, userID int64, on bool) error {
201
if on {
202
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO group_runner_optin (group_id, user_id) VALUES (?, ?)`, groupID, userID)
203
return err
204
}
205
_, err := s.db.ExecContext(ctx, `DELETE FROM group_runner_optin WHERE group_id = ? AND user_id = ?`, groupID, userID)
206
return err
207
}
208
209
// RunnerOptIns zijn de gebruikers die hun runners voor de groep toestaan.
210
func (s *Store) RunnerOptIns(ctx context.Context, groupID int64) ([]int64, error) {
211
rows, err := s.db.QueryContext(ctx, `SELECT user_id FROM group_runner_optin WHERE group_id = ? ORDER BY user_id`, groupID)
212
if err != nil {
213
return nil, err
214
}
215
defer rows.Close()
216
var out []int64
217
for rows.Next() {
218
var id int64
219
if err := rows.Scan(&id); err != nil {
220
return nil, err
221
}
222
out = append(out, id)
223
}
224
return out, rows.Err()
225
}
226
227
// JobRun is de runner-administratie van één job.
228
type JobRun struct {
229
RunnerID int64
230
RunnerName string
231
OfferTier int
232
OfferAt time.Time
233
}
234
235
// JobRun leest de administratie; een lege JobRun als er niets is.
236
func (s *Store) JobRun(ctx context.Context, jobID int64) JobRun {
237
var jr JobRun
238
var at string
239
if err := s.db.QueryRowContext(ctx, `SELECT runner_id, runner_name, offer_tier, offer_at FROM job_runs WHERE job_id = ?`, jobID).Scan(&jr.RunnerID, &jr.RunnerName, &jr.OfferTier, &at); err != nil {
240
return JobRun{}
241
}
242
jr.OfferAt = parseTime(at)
243
return jr
244
}
245
246
// SetJobOffer zet de huidige trede in de runnervolgorde.
247
func (s *Store) SetJobOffer(ctx context.Context, jobID int64, tier int, at time.Time) error {
248
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, offer_tier, offer_at) VALUES (?, ?, ?)
249
ON CONFLICT(job_id) DO UPDATE SET offer_tier = excluded.offer_tier, offer_at = excluded.offer_at`, jobID, tier, at.UTC().Format(time.RFC3339Nano))
250
return err
251
}
252
253
// ClearJobOffer haalt het aanbod weg (bijvoorbeeld bij een retry).
254
func (s *Store) ClearJobOffer(ctx context.Context, jobID int64) error {
255
_, err := s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '', offer_tier = 0 WHERE job_id = ?`, jobID)
256
return err
257
}
258
259
// SetJobRunner noteert wie de job draait.
260
func (s *Store) SetJobRunner(ctx context.Context, jobID, runnerID int64, name string) error {
261
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, runner_id, runner_name) VALUES (?, ?, ?)
262
ON CONFLICT(job_id) DO UPDATE SET runner_id = excluded.runner_id, runner_name = excluded.runner_name`, jobID, runnerID, name)
263
return err
264
}
265
266
// OfferedJobs zijn wachtende jobs met een actief aanbod.
267
func (s *Store) OfferedJobs(ctx context.Context) ([]Job, map[int64]JobRun, error) {
268
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'queued' AND r.offer_at != '' ORDER BY j.id`)
269
if err != nil {
270
return nil, nil, err
271
}
272
var out []Job
273
for rows.Next() {
274
j, err := scanJob(rows)
275
if err != nil {
276
rows.Close()
277
return nil, nil, err
278
}
279
out = append(out, j)
280
}
281
rows.Close()
282
runs := map[int64]JobRun{}
283
for _, j := range out {
284
runs[j.ID] = s.JobRun(ctx, j.ID)
285
}
286
return out, runs, nil
287
}
288
289
// ClaimJob zet een wachtende job op running voor een runner. false als
290
// iemand anders hem al had.
291
func (s *Store) ClaimJob(ctx context.Context, jobID, runnerID int64, name string) (bool, error) {
292
ok, err := s.MarkJobRunning(ctx, jobID)
293
if err != nil || !ok {
294
return false, err
295
}
296
if err := s.SetJobRunner(ctx, jobID, runnerID, name); err != nil {
297
return true, err
298
}
299
_, err = s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '' WHERE job_id = ?`, jobID)
300
return true, err
301
}
302
303
// RemoteJob is een lopende job op een aparte runner.
304
type RemoteJob struct {
305
Job Job
306
RunnerID int64
307
}
308
309
// RemoteRunningJobs geeft lopende jobs die een aparte runner heeft.
310
func (s *Store) RemoteRunningJobs(ctx context.Context) ([]RemoteJob, error) {
311
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'running' AND r.runner_id > 0 ORDER BY j.id`)
312
if err != nil {
313
return nil, err
314
}
315
var out []RemoteJob
316
for rows.Next() {
317
j, err := scanJob(rows)
318
if err != nil {
319
rows.Close()
320
return nil, err
321
}
322
out = append(out, RemoteJob{Job: j})
323
}
324
rows.Close()
325
for i := range out {
326
out[i].RunnerID = s.JobRun(ctx, out[i].Job.ID).RunnerID
327
}
328
return out, nil
329
}
330
331
// JobForRunner geeft een job als die bij runnerID hoort (en loopt of net klaar is).
332
func (s *Store) JobForRunner(ctx context.Context, runnerID, jobID int64) (Job, error) {
333
if s.JobRun(ctx, jobID).RunnerID != runnerID || runnerID == 0 {
334
return Job{}, errNotFound
335
}
336
rows, err := s.db.QueryContext(ctx, jobSelect+` WHERE j.id = ?`, jobID)
337
if err != nil {
338
return Job{}, err
339
}
340
defer rows.Close()
341
if !rows.Next() {
342
return Job{}, errNotFound
343
}
344
return scanJob(rows)
345
}
346
347
// RunnerRunsRepo zegt of de runner nu een job van de repo draait.
348
func (s *Store) RunnerRunsRepo(ctx context.Context, runnerID, repoID int64) bool {
349
var n int
350
_ = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM jobs j JOIN job_runs r ON r.job_id = j.id WHERE r.runner_id = ? AND j.repo_id = ? AND j.status = 'running'`, runnerID, repoID).Scan(&n)
351
return n > 0 && runnerID > 0
352
}
@@ -0,0 +1,352 @@1
package store2
3
import (4
"context"5
"database/sql"6
"errors"7
"strings"8
"time"9
)10
11
// Runner is een aparte bluegit-runner op een eigen machine, gekoppeld aan12
// een account (owner_kind user) of een groep (owner_kind group).13
type Runner struct {14
ID int6415
OwnerKind string16
OwnerID int6417
Name string18
CreatedAt time.Time19
LastSeen time.Time20
Version string21
Revoked bool22
}23
24
// RunnerOnlineWindow: zo lang na de laatste heartbeat telt een runner als online.25
const RunnerOnlineWindow = 90 * time.Second26
27
// Online zegt of de runner recent contact had.28
func (r Runner) Online(now time.Time) bool {29
return !r.Revoked && !r.LastSeen.IsZero() && now.Sub(r.LastSeen) < RunnerOnlineWindow30
}31
32
// Runner-voorkeuren van een account.33
const (34
RunnerPrefOwnFirst = "own_first"35
RunnerPrefBuiltinFirst = "builtin_first"36
)37
38
// ErrRegToken is een onbekend, gebruikt of verlopen registratietoken.39
var ErrRegToken = errors.New("registration token invalid or expired")40
41
// CreateRunnerRegToken maakt een eenmalig registratietoken (24 uur geldig).42
func (s *Store) CreateRunnerRegToken(ctx context.Context, ownerKind string, ownerID, by int64) (string, error) {43
plain, err := newToken("bgrr_")44
if err != nil {45
return "", err46
}47
exp := time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)48
_, err = s.db.ExecContext(ctx, `INSERT INTO runner_reg_tokens (owner_kind, owner_id, token_hash, created_by, expires_at) VALUES (?, ?, ?, ?, ?)`,49
ownerKind, ownerID, hashToken(plain), by, exp)50
return plain, err51
}52
53
// RegisterRunner wisselt een registratietoken in voor een runner en zijn54
// eigen runner-token (bgr_). Alleen de hash wordt bewaard.55
func (s *Store) RegisterRunner(ctx context.Context, regToken, name, version string) (Runner, string, error) {56
tx, err := s.db.BeginTx(ctx, nil)57
if err != nil {58
return Runner{}, "", err59
}60
defer func() { _ = tx.Rollback() }()61
var id, ownerID, by int6462
var kind, exp string63
err = tx.QueryRowContext(ctx, `SELECT id, owner_kind, owner_id, created_by, expires_at FROM runner_reg_tokens WHERE token_hash = ? AND used = 0`, hashToken(regToken)).Scan(&id, &kind, &ownerID, &by, &exp)64
if err != nil || parseTime(exp).Before(time.Now()) {65
return Runner{}, "", ErrRegToken66
}67
if _, err := tx.ExecContext(ctx, `UPDATE runner_reg_tokens SET used = 1 WHERE id = ?`, id); err != nil {68
return Runner{}, "", err69
}70
plain, err := newToken("bgr_")71
if err != nil {72
return Runner{}, "", err73
}74
name = strings.TrimSpace(name)75
if name == "" {76
name = "runner"77
}78
if len(name) > 80 {79
name = name[:80]80
}81
at := now()82
res, err := tx.ExecContext(ctx, `INSERT INTO runners (owner_kind, owner_id, name, token_hash, created_by, created_at, last_seen, version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,83
kind, ownerID, name, hashToken(plain), by, at, at, version)84
if err != nil {85
return Runner{}, "", err86
}87
rid, _ := res.LastInsertId()88
if err := tx.Commit(); err != nil {89
return Runner{}, "", err90
}91
return Runner{ID: rid, OwnerKind: kind, OwnerID: ownerID, Name: name, CreatedAt: parseTime(at), LastSeen: parseTime(at), Version: version}, plain, nil92
}93
94
const runnerCols = `id, owner_kind, owner_id, name, created_at, last_seen, version, revoked`95
96
func scanRunner(sc interface{ Scan(...any) error }) (Runner, error) {97
var r Runner98
var created, seen string99
var revoked int100
if err := sc.Scan(&r.ID, &r.OwnerKind, &r.OwnerID, &r.Name, &created, &seen, &r.Version, &revoked); err != nil {101
return Runner{}, err102
}103
r.CreatedAt, r.LastSeen, r.Revoked = parseTime(created), parseTime(seen), revoked == 1104
return r, nil105
}106
107
// RunnerByToken geeft de runner bij een bgr_-token. Ingetrokken telt niet.108
func (s *Store) RunnerByToken(ctx context.Context, token string) (Runner, error) {109
if !strings.HasPrefix(token, "bgr_") {110
return Runner{}, errNotFound111
}112
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE token_hash = ? AND revoked = 0`, hashToken(token)))113
if errors.Is(err, sql.ErrNoRows) {114
return Runner{}, errNotFound115
}116
return r, err117
}118
119
// RunnerByID geeft één runner.120
func (s *Store) RunnerByID(ctx context.Context, id int64) (Runner, error) {121
r, err := scanRunner(s.db.QueryRowContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE id = ?`, id))122
if errors.Is(err, sql.ErrNoRows) {123
return Runner{}, errNotFound124
}125
return r, err126
}127
128
// TouchRunner is de heartbeat.129
func (s *Store) TouchRunner(ctx context.Context, id int64, version string) error {130
_, err := s.db.ExecContext(ctx, `UPDATE runners SET last_seen = ?, version = CASE WHEN ? = '' THEN version ELSE ? END WHERE id = ?`, now(), version, version, id)131
return err132
}133
134
// ListRunners geeft de niet-ingetrokken runners van een eigenaar.135
func (s *Store) ListRunners(ctx context.Context, ownerKind string, ownerID int64) ([]Runner, error) {136
rows, err := s.db.QueryContext(ctx, `SELECT `+runnerCols+` FROM runners WHERE owner_kind = ? AND owner_id = ? AND revoked = 0 ORDER BY id`, ownerKind, ownerID)137
if err != nil {138
return nil, err139
}140
defer rows.Close()141
var out []Runner142
for rows.Next() {143
r, err := scanRunner(rows)144
if err != nil {145
return nil, err146
}147
out = append(out, r)148
}149
return out, rows.Err()150
}151
152
// RevokeRunner trekt een runner in; zijn token werkt daarna niet meer.153
func (s *Store) RevokeRunner(ctx context.Context, ownerKind string, ownerID, id int64) error {154
res, err := s.db.ExecContext(ctx, `UPDATE runners SET revoked = 1 WHERE id = ? AND owner_kind = ? AND owner_id = ?`, id, ownerKind, ownerID)155
if err != nil {156
return err157
}158
if n, _ := res.RowsAffected(); n == 0 {159
return errNotFound160
}161
return nil162
}163
164
// UserRunnerPref is own_first of builtin_first.165
func (s *Store) UserRunnerPref(ctx context.Context, userID int64) string {166
var p string167
if err := s.db.QueryRowContext(ctx, `SELECT runner_pref FROM users WHERE id = ?`, userID).Scan(&p); err != nil || p != RunnerPrefOwnFirst {168
return RunnerPrefBuiltinFirst169
}170
return p171
}172
173
// SetUserRunnerPref bewaart de voorkeur.174
func (s *Store) SetUserRunnerPref(ctx context.Context, userID int64, pref string) error {175
if pref != RunnerPrefOwnFirst {176
pref = RunnerPrefBuiltinFirst177
}178
_, err := s.db.ExecContext(ctx, `UPDATE users SET runner_pref = ? WHERE id = ?`, pref, userID)179
return err180
}181
182
// GroupBuiltinFirst: "Probeer BlueGit runner eerst".183
func (s *Store) GroupBuiltinFirst(ctx context.Context, groupID int64) bool {184
var v int185
_ = s.db.QueryRowContext(ctx, `SELECT builtin_first FROM groups WHERE id = ?`, groupID).Scan(&v)186
return v == 1187
}188
189
// SetGroupBuiltinFirst zet de groepsinstelling.190
func (s *Store) SetGroupBuiltinFirst(ctx context.Context, groupID int64, on bool) error {191
v := 0192
if on {193
v = 1194
}195
_, err := s.db.ExecContext(ctx, `UPDATE groups SET builtin_first = ? WHERE id = ?`, v, groupID)196
return err197
}198
199
// SetRunnerOptIn: maintainer userID staat zijn account-runners toe voor de groep.200
func (s *Store) SetRunnerOptIn(ctx context.Context, groupID, userID int64, on bool) error {201
if on {202
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO group_runner_optin (group_id, user_id) VALUES (?, ?)`, groupID, userID)203
return err204
}205
_, err := s.db.ExecContext(ctx, `DELETE FROM group_runner_optin WHERE group_id = ? AND user_id = ?`, groupID, userID)206
return err207
}208
209
// RunnerOptIns zijn de gebruikers die hun runners voor de groep toestaan.210
func (s *Store) RunnerOptIns(ctx context.Context, groupID int64) ([]int64, error) {211
rows, err := s.db.QueryContext(ctx, `SELECT user_id FROM group_runner_optin WHERE group_id = ? ORDER BY user_id`, groupID)212
if err != nil {213
return nil, err214
}215
defer rows.Close()216
var out []int64217
for rows.Next() {218
var id int64219
if err := rows.Scan(&id); err != nil {220
return nil, err221
}222
out = append(out, id)223
}224
return out, rows.Err()225
}226
227
// JobRun is de runner-administratie van één job.228
type JobRun struct {229
RunnerID int64230
RunnerName string231
OfferTier int232
OfferAt time.Time233
}234
235
// JobRun leest de administratie; een lege JobRun als er niets is.236
func (s *Store) JobRun(ctx context.Context, jobID int64) JobRun {237
var jr JobRun238
var at string239
if err := s.db.QueryRowContext(ctx, `SELECT runner_id, runner_name, offer_tier, offer_at FROM job_runs WHERE job_id = ?`, jobID).Scan(&jr.RunnerID, &jr.RunnerName, &jr.OfferTier, &at); err != nil {240
return JobRun{}241
}242
jr.OfferAt = parseTime(at)243
return jr244
}245
246
// SetJobOffer zet de huidige trede in de runnervolgorde.247
func (s *Store) SetJobOffer(ctx context.Context, jobID int64, tier int, at time.Time) error {248
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, offer_tier, offer_at) VALUES (?, ?, ?)249
ON CONFLICT(job_id) DO UPDATE SET offer_tier = excluded.offer_tier, offer_at = excluded.offer_at`, jobID, tier, at.UTC().Format(time.RFC3339Nano))250
return err251
}252
253
// ClearJobOffer haalt het aanbod weg (bijvoorbeeld bij een retry).254
func (s *Store) ClearJobOffer(ctx context.Context, jobID int64) error {255
_, err := s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '', offer_tier = 0 WHERE job_id = ?`, jobID)256
return err257
}258
259
// SetJobRunner noteert wie de job draait.260
func (s *Store) SetJobRunner(ctx context.Context, jobID, runnerID int64, name string) error {261
_, err := s.db.ExecContext(ctx, `INSERT INTO job_runs (job_id, runner_id, runner_name) VALUES (?, ?, ?)262
ON CONFLICT(job_id) DO UPDATE SET runner_id = excluded.runner_id, runner_name = excluded.runner_name`, jobID, runnerID, name)263
return err264
}265
266
// OfferedJobs zijn wachtende jobs met een actief aanbod.267
func (s *Store) OfferedJobs(ctx context.Context) ([]Job, map[int64]JobRun, error) {268
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'queued' AND r.offer_at != '' ORDER BY j.id`)269
if err != nil {270
return nil, nil, err271
}272
var out []Job273
for rows.Next() {274
j, err := scanJob(rows)275
if err != nil {276
rows.Close()277
return nil, nil, err278
}279
out = append(out, j)280
}281
rows.Close()282
runs := map[int64]JobRun{}283
for _, j := range out {284
runs[j.ID] = s.JobRun(ctx, j.ID)285
}286
return out, runs, nil287
}288
289
// ClaimJob zet een wachtende job op running voor een runner. false als290
// iemand anders hem al had.291
func (s *Store) ClaimJob(ctx context.Context, jobID, runnerID int64, name string) (bool, error) {292
ok, err := s.MarkJobRunning(ctx, jobID)293
if err != nil || !ok {294
return false, err295
}296
if err := s.SetJobRunner(ctx, jobID, runnerID, name); err != nil {297
return true, err298
}299
_, err = s.db.ExecContext(ctx, `UPDATE job_runs SET offer_at = '' WHERE job_id = ?`, jobID)300
return true, err301
}302
303
// RemoteJob is een lopende job op een aparte runner.304
type RemoteJob struct {305
Job Job306
RunnerID int64307
}308
309
// RemoteRunningJobs geeft lopende jobs die een aparte runner heeft.310
func (s *Store) RemoteRunningJobs(ctx context.Context) ([]RemoteJob, error) {311
rows, err := s.db.QueryContext(ctx, jobSelect+` JOIN job_runs r ON r.job_id = j.id WHERE j.status = 'running' AND r.runner_id > 0 ORDER BY j.id`)312
if err != nil {313
return nil, err314
}315
var out []RemoteJob316
for rows.Next() {317
j, err := scanJob(rows)318
if err != nil {319
rows.Close()320
return nil, err321
}322
out = append(out, RemoteJob{Job: j})323
}324
rows.Close()325
for i := range out {326
out[i].RunnerID = s.JobRun(ctx, out[i].Job.ID).RunnerID327
}328
return out, nil329
}330
331
// JobForRunner geeft een job als die bij runnerID hoort (en loopt of net klaar is).332
func (s *Store) JobForRunner(ctx context.Context, runnerID, jobID int64) (Job, error) {333
if s.JobRun(ctx, jobID).RunnerID != runnerID || runnerID == 0 {334
return Job{}, errNotFound335
}336
rows, err := s.db.QueryContext(ctx, jobSelect+` WHERE j.id = ?`, jobID)337
if err != nil {338
return Job{}, err339
}340
defer rows.Close()341
if !rows.Next() {342
return Job{}, errNotFound343
}344
return scanJob(rows)345
}346
347
// RunnerRunsRepo zegt of de runner nu een job van de repo draait.348
func (s *Store) RunnerRunsRepo(ctx context.Context, runnerID, repoID int64) bool {349
var n int350
_ = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM jobs j JOIN job_runs r ON r.job_id = j.id WHERE r.runner_id = ? AND j.repo_id = ? AND j.status = 'running'`, runnerID, repoID).Scan(&n)351
return n > 0 && runnerID > 0352
}@@ -102,6 +102,8 @@ func (s *Store) migrate() error {
102
102
`ALTER TABLE pipelines ADD COLUMN target_sha TEXT NOT NULL DEFAULT ''`,
103
103
`ALTER TABLE users ADD COLUMN display_name TEXT NOT NULL DEFAULT ''`,
104
104
`ALTER TABLE users ADD COLUMN bio TEXT NOT NULL DEFAULT ''`,
105
`ALTER TABLE users ADD COLUMN runner_pref TEXT NOT NULL DEFAULT 'builtin_first'`,
106
`ALTER TABLE groups ADD COLUMN builtin_first INTEGER NOT NULL DEFAULT 0`,
105
107
}
106
108
for _, q := range alters {
107
109
if _, err := s.db.Exec(q); err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column") {
@@ -115,6 +117,39 @@ func (s *Store) migrate() error {
115
117
}
116
118
117
119
const schema = `
120
CREATE TABLE IF NOT EXISTS runners (
121
id INTEGER PRIMARY KEY,
122
owner_kind TEXT NOT NULL,
123
owner_id INTEGER NOT NULL,
124
name TEXT NOT NULL,
125
token_hash TEXT NOT NULL UNIQUE,
126
created_by INTEGER NOT NULL DEFAULT 0,
127
created_at TEXT NOT NULL,
128
last_seen TEXT NOT NULL DEFAULT '',
129
version TEXT NOT NULL DEFAULT '',
130
revoked INTEGER NOT NULL DEFAULT 0
131
);
132
CREATE TABLE IF NOT EXISTS runner_reg_tokens (
133
id INTEGER PRIMARY KEY,
134
owner_kind TEXT NOT NULL,
135
owner_id INTEGER NOT NULL,
136
token_hash TEXT NOT NULL UNIQUE,
137
created_by INTEGER NOT NULL,
138
expires_at TEXT NOT NULL,
139
used INTEGER NOT NULL DEFAULT 0
140
);
141
CREATE TABLE IF NOT EXISTS group_runner_optin (
142
group_id INTEGER NOT NULL,
143
user_id INTEGER NOT NULL,
144
PRIMARY KEY (group_id, user_id)
145
);
146
CREATE TABLE IF NOT EXISTS job_runs (
147
job_id INTEGER PRIMARY KEY,
148
runner_id INTEGER NOT NULL DEFAULT 0,
149
runner_name TEXT NOT NULL DEFAULT '',
150
offer_tier INTEGER NOT NULL DEFAULT 0,
151
offer_at TEXT NOT NULL DEFAULT ''
152
);
118
153
CREATE TABLE IF NOT EXISTS users (
119
154
id INTEGER PRIMARY KEY,
120
155
auth_id INTEGER NOT NULL UNIQUE,
@@ -102,6 +102,8 @@ func (s *Store) migrate() error {102
`ALTER TABLE pipelines ADD COLUMN target_sha TEXT NOT NULL DEFAULT ''`,102
`ALTER TABLE pipelines ADD COLUMN target_sha TEXT NOT NULL DEFAULT ''`,103
`ALTER TABLE users ADD COLUMN display_name TEXT NOT NULL DEFAULT ''`,103
`ALTER TABLE users ADD COLUMN display_name TEXT NOT NULL DEFAULT ''`,104
`ALTER TABLE users ADD COLUMN bio TEXT NOT NULL DEFAULT ''`,104
`ALTER TABLE users ADD COLUMN bio TEXT NOT NULL DEFAULT ''`,105
`ALTER TABLE users ADD COLUMN runner_pref TEXT NOT NULL DEFAULT 'builtin_first'`,106
`ALTER TABLE groups ADD COLUMN builtin_first INTEGER NOT NULL DEFAULT 0`,105
}107
}106
for _, q := range alters {108
for _, q := range alters {107
if _, err := s.db.Exec(q); err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column") {109
if _, err := s.db.Exec(q); err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column") {@@ -115,6 +117,39 @@ func (s *Store) migrate() error {115
}117
}116
118
117
const schema = `119
const schema = `120
CREATE TABLE IF NOT EXISTS runners (121
id INTEGER PRIMARY KEY,122
owner_kind TEXT NOT NULL,123
owner_id INTEGER NOT NULL,124
name TEXT NOT NULL,125
token_hash TEXT NOT NULL UNIQUE,126
created_by INTEGER NOT NULL DEFAULT 0,127
created_at TEXT NOT NULL,128
last_seen TEXT NOT NULL DEFAULT '',129
version TEXT NOT NULL DEFAULT '',130
revoked INTEGER NOT NULL DEFAULT 0131
);132
CREATE TABLE IF NOT EXISTS runner_reg_tokens (133
id INTEGER PRIMARY KEY,134
owner_kind TEXT NOT NULL,135
owner_id INTEGER NOT NULL,136
token_hash TEXT NOT NULL UNIQUE,137
created_by INTEGER NOT NULL,138
expires_at TEXT NOT NULL,139
used INTEGER NOT NULL DEFAULT 0140
);141
CREATE TABLE IF NOT EXISTS group_runner_optin (142
group_id INTEGER NOT NULL,143
user_id INTEGER NOT NULL,144
PRIMARY KEY (group_id, user_id)145
);146
CREATE TABLE IF NOT EXISTS job_runs (147
job_id INTEGER PRIMARY KEY,148
runner_id INTEGER NOT NULL DEFAULT 0,149
runner_name TEXT NOT NULL DEFAULT '',150
offer_tier INTEGER NOT NULL DEFAULT 0,151
offer_at TEXT NOT NULL DEFAULT ''152
);118
CREATE TABLE IF NOT EXISTS users (153
CREATE TABLE IF NOT EXISTS users (119
id INTEGER PRIMARY KEY,154
id INTEGER PRIMARY KEY,120
auth_id INTEGER NOT NULL UNIQUE,155
auth_id INTEGER NOT NULL UNIQUE,@@ -0,0 +1,35 @@
1
# Maintainer: Tim Falken <tim@timfalken.com>
2
pkgname=bluegit-runner
3
pkgver=20261024090636
4
pkgrel=1
5
pkgdesc='Separate CI runner for BlueGit (runs jobs in Docker on your own machine)'
6
arch=('x86_64' 'aarch64')
7
url='https://git.timfalken.com/timfa/bluegit'
8
license=('MIT')
9
depends=('git' 'docker')
10
makedepends=('go')
11
options=('!debug')
12
install=bluegit-runner.install
13
_commit=6a359db00000000000000000000000000000000a
14
source=("bluegit-${_commit}.tar.gz::https://git.timfalken.com/timfa/bluegit/-/archive/${_commit}/bluegit-${_commit}.tar.gz")
15
sha256sums=('0000000000000000000000000000000000000000000000000000000000000000')
16
17
build() {
18
cd "$srcdir/bluegit-${_commit}"
19
export CGO_ENABLED=0
20
export GOFLAGS='-trimpath'
21
export GOCACHE="${BUILDDIR:-$srcdir}/gocache"
22
export GOPATH="${BUILDDIR:-$srcdir}/gopath"
23
go build -ldflags="-s -w -X main.version=${pkgver}" -o bluegit-runner ./cmd/bluegit-runner
24
}
25
26
package() {
27
cd "$srcdir/bluegit-${_commit}"
28
install -Dm755 bluegit-runner "$pkgdir/usr/bin/bluegit-runner"
29
install -Dm644 packaging/runner/bluegit-runner.service "$pkgdir/usr/lib/systemd/system/bluegit-runner.service"
30
install -Dm644 packaging/runner/sysusers.conf "$pkgdir/usr/lib/sysusers.d/bluegit-runner.conf"
31
install -Dm644 packaging/runner/tmpfiles.conf "$pkgdir/usr/lib/tmpfiles.d/bluegit-runner.conf"
32
install -Dm644 packaging/runner/config.yml.example "$pkgdir/usr/share/bluegit-runner/config.yml.example"
33
install -Dm644 docs/runner.md "$pkgdir/usr/share/doc/bluegit-runner/README.md"
34
install -Dm644 LICENSE "$pkgdir/usr/share/licenses/$pkgname/LICENSE"
35
}
@@ -0,0 +1,35 @@1
# Maintainer: Tim Falken <tim@timfalken.com>2
pkgname=bluegit-runner3
pkgver=202610240906364
pkgrel=15
pkgdesc='Separate CI runner for BlueGit (runs jobs in Docker on your own machine)'6
arch=('x86_64' 'aarch64')7
url='https://git.timfalken.com/timfa/bluegit'8
license=('MIT')9
depends=('git' 'docker')10
makedepends=('go')11
options=('!debug')12
install=bluegit-runner.install13
_commit=6a359db00000000000000000000000000000000a14
source=("bluegit-${_commit}.tar.gz::https://git.timfalken.com/timfa/bluegit/-/archive/${_commit}/bluegit-${_commit}.tar.gz")15
sha256sums=('0000000000000000000000000000000000000000000000000000000000000000')16
17
build() {18
cd "$srcdir/bluegit-${_commit}"19
export CGO_ENABLED=020
export GOFLAGS='-trimpath'21
export GOCACHE="${BUILDDIR:-$srcdir}/gocache"22
export GOPATH="${BUILDDIR:-$srcdir}/gopath"23
go build -ldflags="-s -w -X main.version=${pkgver}" -o bluegit-runner ./cmd/bluegit-runner24
}25
26
package() {27
cd "$srcdir/bluegit-${_commit}"28
install -Dm755 bluegit-runner "$pkgdir/usr/bin/bluegit-runner"29
install -Dm644 packaging/runner/bluegit-runner.service "$pkgdir/usr/lib/systemd/system/bluegit-runner.service"30
install -Dm644 packaging/runner/sysusers.conf "$pkgdir/usr/lib/sysusers.d/bluegit-runner.conf"31
install -Dm644 packaging/runner/tmpfiles.conf "$pkgdir/usr/lib/tmpfiles.d/bluegit-runner.conf"32
install -Dm644 packaging/runner/config.yml.example "$pkgdir/usr/share/bluegit-runner/config.yml.example"33
install -Dm644 docs/runner.md "$pkgdir/usr/share/doc/bluegit-runner/README.md"34
install -Dm644 LICENSE "$pkgdir/usr/share/licenses/$pkgname/LICENSE"35
}@@ -0,0 +1,43 @@
1
# bluegit-runner: systemd-unit met User=bluegit-runner (groep docker).
2
# Na installatie draait de service zodra er een config is
3
# (bluegit-runner register). Een upgrade herstart de service.
4
# Zonder draaiende systemd (chroot of container) faalt de transactie niet.
5
6
_bgr_systemctl() {
7
if ! command -v systemctl >/dev/null 2>&1 || [[ ! -d /run/systemd/system ]]; then
8
echo "bluegit-runner: systemd draait niet; de service is niet aangepast."
9
return 0
10
fi
11
systemctl "$@" || true
12
}
13
14
post_install() {
15
systemd-sysusers bluegit-runner.conf || true
16
systemd-tmpfiles --create bluegit-runner.conf || true
17
_bgr_systemctl daemon-reload
18
_bgr_systemctl enable --now bluegit-runner.service
19
if [[ ! -f /etc/bluegit-runner/config.yml ]]; then
20
cat <<'MSG'
21
bluegit-runner: register this machine to start the service:
22
1. In BlueGit: Settings → Runners → "New registration token"
23
(or a group's Settings → Runners).
24
2. sudo bluegit-runner register --url https://your-bluegit --token bgrr_...
25
The service is enabled and starts as soon as the config exists.
26
MSG
27
fi
28
}
29
30
post_upgrade() {
31
systemd-sysusers bluegit-runner.conf || true
32
systemd-tmpfiles --create bluegit-runner.conf || true
33
_bgr_systemctl daemon-reload
34
_bgr_systemctl try-restart bluegit-runner.service
35
}
36
37
pre_remove() {
38
_bgr_systemctl disable --now bluegit-runner.service
39
}
40
41
post_remove() {
42
_bgr_systemctl daemon-reload
43
}
@@ -0,0 +1,43 @@1
# bluegit-runner: systemd-unit met User=bluegit-runner (groep docker).2
# Na installatie draait de service zodra er een config is3
# (bluegit-runner register). Een upgrade herstart de service.4
# Zonder draaiende systemd (chroot of container) faalt de transactie niet.5
6
_bgr_systemctl() {7
if ! command -v systemctl >/dev/null 2>&1 || [[ ! -d /run/systemd/system ]]; then8
echo "bluegit-runner: systemd draait niet; de service is niet aangepast."9
return 010
fi11
systemctl "$@" || true12
}13
14
post_install() {15
systemd-sysusers bluegit-runner.conf || true16
systemd-tmpfiles --create bluegit-runner.conf || true17
_bgr_systemctl daemon-reload18
_bgr_systemctl enable --now bluegit-runner.service19
if [[ ! -f /etc/bluegit-runner/config.yml ]]; then20
cat <<'MSG'21
bluegit-runner: register this machine to start the service:22
1. In BlueGit: Settings → Runners → "New registration token"23
(or a group's Settings → Runners).24
2. sudo bluegit-runner register --url https://your-bluegit --token bgrr_...25
The service is enabled and starts as soon as the config exists.26
MSG27
fi28
}29
30
post_upgrade() {31
systemd-sysusers bluegit-runner.conf || true32
systemd-tmpfiles --create bluegit-runner.conf || true33
_bgr_systemctl daemon-reload34
_bgr_systemctl try-restart bluegit-runner.service35
}36
37
pre_remove() {38
_bgr_systemctl disable --now bluegit-runner.service39
}40
41
post_remove() {42
_bgr_systemctl daemon-reload43
}@@ -0,0 +1,26 @@
1
[Unit]
2
Description=BlueGit runner (separate CI runner)
3
Documentation=https://git.timfalken.com/timfa/bluegit/-/blob/master/docs/runner.md
4
After=network-online.target docker.service
5
Wants=network-online.target
6
# Pas na "bluegit-runner register" is er een config.
7
ConditionPathExists=/etc/bluegit-runner/config.yml
8
9
[Service]
10
Type=simple
11
User=bluegit-runner
12
Group=bluegit-runner
13
# Jobs draaien in Docker-containers via de socket (root:docker).
14
SupplementaryGroups=docker
15
ExecStart=/usr/bin/bluegit-runner run --config /etc/bluegit-runner/config.yml
16
Restart=always
17
RestartSec=10
18
NoNewPrivileges=true
19
ProtectSystem=strict
20
ProtectHome=true
21
StateDirectory=bluegit-runner
22
ReadWritePaths=/var/lib/bluegit-runner
23
PrivateTmp=true
24
25
[Install]
26
WantedBy=multi-user.target
@@ -0,0 +1,26 @@1
[Unit]2
Description=BlueGit runner (separate CI runner)3
Documentation=https://git.timfalken.com/timfa/bluegit/-/blob/master/docs/runner.md4
After=network-online.target docker.service5
Wants=network-online.target6
# Pas na "bluegit-runner register" is er een config.7
ConditionPathExists=/etc/bluegit-runner/config.yml8
9
[Service]10
Type=simple11
User=bluegit-runner12
Group=bluegit-runner13
# Jobs draaien in Docker-containers via de socket (root:docker).14
SupplementaryGroups=docker15
ExecStart=/usr/bin/bluegit-runner run --config /etc/bluegit-runner/config.yml16
Restart=always17
RestartSec=1018
NoNewPrivileges=true19
ProtectSystem=strict20
ProtectHome=true21
StateDirectory=bluegit-runner22
ReadWritePaths=/var/lib/bluegit-runner23
PrivateTmp=true24
25
[Install]26
WantedBy=multi-user.target@@ -0,0 +1,12 @@
1
# /etc/bluegit-runner/config.yml — written by "bluegit-runner register".
2
# url and token are filled in by register; the token is the runner's own
3
# token (bgr_...), only its hash is stored on the server.
4
url: https://git.example.com
5
token: ""
6
name: my-machine
7
# Optional:
8
# docker: docker
9
# git: git
10
# work_dir: /var/lib/bluegit-runner
11
# network: bridge
12
# concurrency: 1
@@ -0,0 +1,12 @@1
# /etc/bluegit-runner/config.yml — written by "bluegit-runner register".2
# url and token are filled in by register; the token is the runner's own3
# token (bgr_...), only its hash is stored on the server.4
url: https://git.example.com5
token: ""6
name: my-machine7
# Optional:8
# docker: docker9
# git: git10
# work_dir: /var/lib/bluegit-runner11
# network: bridge12
# concurrency: 1@@ -0,0 +1,2 @@
1
u bluegit-runner - "BlueGit runner" /var/lib/bluegit-runner /usr/bin/nologin
2
m bluegit-runner docker
@@ -0,0 +1,2 @@1
u bluegit-runner - "BlueGit runner" /var/lib/bluegit-runner /usr/bin/nologin2
m bluegit-runner docker@@ -0,0 +1,2 @@
1
d /etc/bluegit-runner 0750 root bluegit-runner -
2
d /var/lib/bluegit-runner 0750 bluegit-runner bluegit-runner -
@@ -0,0 +1,2 @@1
d /etc/bluegit-runner 0750 root bluegit-runner -2
d /var/lib/bluegit-runner 0750 bluegit-runner bluegit-runner -@@ -1,6 +1,8 @@
1
1
#!/usr/bin/env bash
2
# Update the AUR package bluegit from the BlueGit source archive of this commit.
3
# Pushes ssh://aur@aur.archlinux.org/bluegit.git master. AUR only accepts master.
2
# Update an AUR package from the BlueGit source archive of this commit.
3
# AUR_PKG=bluegit (default, packaging/aur) or AUR_PKG=bluegit-runner
4
# (packaging/aur-runner). Pushes ssh://aur@aur.archlinux.org/$AUR_PKG.git
5
# master. AUR only accepts master.
4
6
set -euo pipefail
5
7
6
8
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -88,6 +90,14 @@ else
88
90
fi
89
91
VERSION="$(pkgver_of_commit "$ROOT" "$COMMIT")"
90
92
93
PKG="${AUR_PKG:-bluegit}"
94
case "$PKG" in
95
bluegit) PKGDIR="$ROOT/packaging/aur" ;;
96
bluegit-runner) PKGDIR="$ROOT/packaging/aur-runner" ;;
97
*) echo "Unknown AUR_PKG: $PKG" >&2; exit 1 ;;
98
esac
99
INSTALL_FILE="${PKG}.install"
100
91
101
TMPDIR="${TMPDIR:-/tmp}"
92
102
WORKDIR="$(mktemp -d "$TMPDIR/bluegit-aur-XXXXXX")"
93
103
cleanup() { rm -rf "$WORKDIR"; }
@@ -102,26 +112,26 @@ if ! curl -fsSL "$SOURCE_URL" -o "$TMP_SOURCE"; then
102
112
fi
103
113
SOURCE_SHA="$(sha256sum "$TMP_SOURCE" | awk '{print $1}')"
104
114
105
dest="$WORKDIR/bluegit"
106
clone_or_init_aur_repo "bluegit" "$dest"
115
dest="$WORKDIR/$PKG"
116
clone_or_init_aur_repo "$PKG" "$dest"
107
117
cd "$dest"
108
118
git config user.name "${AUR_GIT_AUTHOR_NAME:-BlueGit release bot}"
109
119
git config user.email "${AUR_GIT_AUTHOR_EMAIL:-noreply@example.invalid}"
110
cp "$ROOT/packaging/aur/PKGBUILD" PKGBUILD
111
cp "$ROOT/packaging/aur/bluegit.install" bluegit.install
120
cp "$PKGDIR/PKGBUILD" PKGBUILD
121
cp "$PKGDIR/$INSTALL_FILE" "$INSTALL_FILE"
112
122
rewrite_pkgbuild "$VERSION" "$COMMIT" "$SOURCE_SHA" PKGBUILD
113
123
generate_srcinfo
114
124
115
if git diff --quiet -- PKGBUILD .SRCINFO bluegit.install 2>/dev/null && git rev-parse --verify HEAD >/dev/null 2>&1; then
116
echo "AUR repo already up to date: bluegit ${VERSION}"
125
if git diff --quiet -- PKGBUILD .SRCINFO "$INSTALL_FILE" 2>/dev/null && git rev-parse --verify HEAD >/dev/null 2>&1; then
126
echo "AUR repo already up to date: ${PKG} ${VERSION}"
117
127
exit 0
118
128
fi
119
129
120
git add PKGBUILD .SRCINFO bluegit.install
130
git add PKGBUILD .SRCINFO "$INSTALL_FILE"
121
131
if git rev-parse --verify HEAD >/dev/null 2>&1; then
122
132
git commit -m "upstream ${VERSION}"
123
133
else
124
134
git commit -m "initial import ${VERSION}"
125
135
fi
126
136
git push -u origin master
127
echo "AUR updated: bluegit ${VERSION}"
137
echo "AUR updated: ${PKG} ${VERSION}"
@@ -1,6 +1,8 @@1
#!/usr/bin/env bash1
#!/usr/bin/env bash2
# Update the AUR package bluegit from the BlueGit source archive of this commit.2
# Update an AUR package from the BlueGit source archive of this commit.3
# Pushes ssh://aur@aur.archlinux.org/bluegit.git master. AUR only accepts master.3
# AUR_PKG=bluegit (default, packaging/aur) or AUR_PKG=bluegit-runner4
# (packaging/aur-runner). Pushes ssh://aur@aur.archlinux.org/$AUR_PKG.git5
# master. AUR only accepts master.4
set -euo pipefail6
set -euo pipefail5
7
6
ROOT="$(cd "$(dirname "$0")/.." && pwd)"8
ROOT="$(cd "$(dirname "$0")/.." && pwd)"@@ -88,6 +90,14 @@ else88
fi90
fi89
VERSION="$(pkgver_of_commit "$ROOT" "$COMMIT")"91
VERSION="$(pkgver_of_commit "$ROOT" "$COMMIT")"90
92
93
PKG="${AUR_PKG:-bluegit}"94
case "$PKG" in95
bluegit) PKGDIR="$ROOT/packaging/aur" ;;96
bluegit-runner) PKGDIR="$ROOT/packaging/aur-runner" ;;97
*) echo "Unknown AUR_PKG: $PKG" >&2; exit 1 ;;98
esac99
INSTALL_FILE="${PKG}.install"100
91
TMPDIR="${TMPDIR:-/tmp}"101
TMPDIR="${TMPDIR:-/tmp}"92
WORKDIR="$(mktemp -d "$TMPDIR/bluegit-aur-XXXXXX")"102
WORKDIR="$(mktemp -d "$TMPDIR/bluegit-aur-XXXXXX")"93
cleanup() { rm -rf "$WORKDIR"; }103
cleanup() { rm -rf "$WORKDIR"; }@@ -102,26 +112,26 @@ if ! curl -fsSL "$SOURCE_URL" -o "$TMP_SOURCE"; then102
fi112
fi103
SOURCE_SHA="$(sha256sum "$TMP_SOURCE" | awk '{print $1}')"113
SOURCE_SHA="$(sha256sum "$TMP_SOURCE" | awk '{print $1}')"104
114
105
dest="$WORKDIR/bluegit"115
dest="$WORKDIR/$PKG"106
clone_or_init_aur_repo "bluegit" "$dest"116
clone_or_init_aur_repo "$PKG" "$dest"107
cd "$dest"117
cd "$dest"108
git config user.name "${AUR_GIT_AUTHOR_NAME:-BlueGit release bot}"118
git config user.name "${AUR_GIT_AUTHOR_NAME:-BlueGit release bot}"109
git config user.email "${AUR_GIT_AUTHOR_EMAIL:-noreply@example.invalid}"119
git config user.email "${AUR_GIT_AUTHOR_EMAIL:-noreply@example.invalid}"110
cp "$ROOT/packaging/aur/PKGBUILD" PKGBUILD120
cp "$PKGDIR/PKGBUILD" PKGBUILD111
cp "$ROOT/packaging/aur/bluegit.install" bluegit.install121
cp "$PKGDIR/$INSTALL_FILE" "$INSTALL_FILE"112
rewrite_pkgbuild "$VERSION" "$COMMIT" "$SOURCE_SHA" PKGBUILD122
rewrite_pkgbuild "$VERSION" "$COMMIT" "$SOURCE_SHA" PKGBUILD113
generate_srcinfo123
generate_srcinfo114
124
115
if git diff --quiet -- PKGBUILD .SRCINFO bluegit.install 2>/dev/null && git rev-parse --verify HEAD >/dev/null 2>&1; then125
if git diff --quiet -- PKGBUILD .SRCINFO "$INSTALL_FILE" 2>/dev/null && git rev-parse --verify HEAD >/dev/null 2>&1; then116
echo "AUR repo already up to date: bluegit ${VERSION}"126
echo "AUR repo already up to date: ${PKG} ${VERSION}"117
exit 0127
exit 0118
fi128
fi119
129
120
git add PKGBUILD .SRCINFO bluegit.install130
git add PKGBUILD .SRCINFO "$INSTALL_FILE"121
if git rev-parse --verify HEAD >/dev/null 2>&1; then131
if git rev-parse --verify HEAD >/dev/null 2>&1; then122
git commit -m "upstream ${VERSION}"132
git commit -m "upstream ${VERSION}"123
else133
else124
git commit -m "initial import ${VERSION}"134
git commit -m "initial import ${VERSION}"125
fi135
fi126
git push -u origin master136
git push -u origin master127
echo "AUR updated: bluegit ${VERSION}"137
echo "AUR updated: ${PKG} ${VERSION}"