| 1 | //go:build integration |
| 2 | |
| 3 | // Integratietests: elke test start een echte BlueGit (BLUEGIT_BIN), maakt |
| 4 | // daar een beheerder, token en Backupserver-Key aan, en test de backup |
| 5 | // server ertegen. Zonder BLUEGIT_BIN falen ze; ze slaan niet over, zodat de |
| 6 | // release-jobs nooit op een overgeslagen test kunnen leunen. |
| 7 | package integration |
| 8 | |
| 9 | import ( |
| 10 | "bytes" |
| 11 | "context" |
| 12 | "encoding/json" |
| 13 | "fmt" |
| 14 | "io" |
| 15 | "log/slog" |
| 16 | "net" |
| 17 | "net/http" |
| 18 | "net/http/cookiejar" |
| 19 | "net/url" |
| 20 | "os" |
| 21 | "os/exec" |
| 22 | "path/filepath" |
| 23 | "strings" |
| 24 | "sync" |
| 25 | "testing" |
| 26 | "time" |
| 27 | |
| 28 | "git.timfalken.com/timfa/bluegit-backup/internal/config" |
| 29 | "git.timfalken.com/timfa/bluegit-backup/internal/server" |
| 30 | ) |
| 31 | |
| 32 | func bluegitBin(t *testing.T) string { |
| 33 | t.Helper() |
| 34 | bin := os.Getenv("BLUEGIT_BIN") |
| 35 | if bin == "" { |
| 36 | t.Fatal("BLUEGIT_BIN ontbreekt: bouw BlueGit met scripts/build-bluegit.sh") |
| 37 | } |
| 38 | if _, err := os.Stat(bin); err != nil { |
| 39 | t.Fatalf("BLUEGIT_BIN: %v", err) |
| 40 | } |
| 41 | if _, err := exec.LookPath("git"); err != nil { |
| 42 | t.Fatal("git ontbreekt") |
| 43 | } |
| 44 | return bin |
| 45 | } |
| 46 | |
| 47 | func freePort(t *testing.T) int { |
| 48 | t.Helper() |
| 49 | ln, err := net.Listen("tcp", "127.0.0.1:0") |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | defer ln.Close() |
| 54 | return ln.Addr().(*net.TCPAddr).Port |
| 55 | } |
| 56 | |
| 57 | // BlueGit is één lokaal opgespinde BlueGit. |
| 58 | type BlueGit struct { |
| 59 | t *testing.T |
| 60 | Bin string |
| 61 | URL string |
| 62 | Config string |
| 63 | State string |
| 64 | Storage string |
| 65 | Token string |
| 66 | cmd *exec.Cmd |
| 67 | log *syncBuf |
| 68 | // Server is het id van de laatst toegevoegde backup server in BlueGit. |
| 69 | Server string |
| 70 | } |
| 71 | |
| 72 | type syncBuf struct { |
| 73 | mu sync.Mutex |
| 74 | b bytes.Buffer |
| 75 | } |
| 76 | |
| 77 | func (s *syncBuf) Write(p []byte) (int, error) { |
| 78 | s.mu.Lock() |
| 79 | defer s.mu.Unlock() |
| 80 | return s.b.Write(p) |
| 81 | } |
| 82 | |
| 83 | func (s *syncBuf) String() string { |
| 84 | s.mu.Lock() |
| 85 | defer s.mu.Unlock() |
| 86 | return s.b.String() |
| 87 | } |
| 88 | |
| 89 | func startBlueGit(t *testing.T) *BlueGit { |
| 90 | t.Helper() |
| 91 | bin := bluegitBin(t) |
| 92 | dir := t.TempDir() |
| 93 | port := freePort(t) |
| 94 | g := &BlueGit{t: t, Bin: bin, URL: fmt.Sprintf("http://127.0.0.1:%d", port), |
| 95 | State: filepath.Join(dir, "state"), Storage: filepath.Join(dir, "storage"), Config: filepath.Join(dir, "bluegit.yml"), log: &syncBuf{}} |
| 96 | _ = os.MkdirAll(g.Storage, 0o750) |
| 97 | cfg := fmt.Sprintf(`base_url: %s |
| 98 | listen: 127.0.0.1:%d |
| 99 | state_path: %s |
| 100 | storage_path: %s |
| 101 | auth: |
| 102 | mode: dev |
| 103 | dev_user: tim |
| 104 | dev_auth_id: 1 |
| 105 | user_agent: bluegit-backup-it |
| 106 | git: |
| 107 | binary: git |
| 108 | default_branch: master |
| 109 | ssh: |
| 110 | enabled: false |
| 111 | ci: |
| 112 | enabled: false |
| 113 | `, g.URL, port, g.State, g.Storage) |
| 114 | if err := os.WriteFile(g.Config, []byte(cfg), 0o600); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | g.cmd = exec.Command(bin, "serve", "--config", g.Config) |
| 118 | g.cmd.Stdout, g.cmd.Stderr = g.log, g.log |
| 119 | if err := g.cmd.Start(); err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | t.Cleanup(func() { |
| 123 | _ = g.cmd.Process.Kill() |
| 124 | _, _ = g.cmd.Process.Wait() |
| 125 | if t.Failed() { |
| 126 | t.Logf("BlueGit-log %s:\n%s", g.URL, tail(g.log.String(), 60)) |
| 127 | } |
| 128 | }) |
| 129 | g.WaitUp(20 * time.Second) |
| 130 | // Aanmelden maakt de eerste gebruiker (beheerder) aan. |
| 131 | jar, _ := cookiejar.New(nil) |
| 132 | hc := &http.Client{Jar: jar} |
| 133 | res, err := hc.Get(g.URL + "/login") |
| 134 | if err != nil { |
| 135 | t.Fatal(err) |
| 136 | } |
| 137 | res.Body.Close() |
| 138 | out, err := exec.Command(bin, "token", "create", "--config", g.Config, "--user", "tim", "--name", "it").CombinedOutput() |
| 139 | if err != nil { |
| 140 | t.Fatalf("token: %v %s", err, out) |
| 141 | } |
| 142 | g.Token = strings.TrimSpace(string(out)) |
| 143 | return g |
| 144 | } |
| 145 | |
| 146 | func tail(s string, n int) string { |
| 147 | lines := strings.Split(s, "\n") |
| 148 | if len(lines) > n { |
| 149 | lines = lines[len(lines)-n:] |
| 150 | } |
| 151 | return strings.Join(lines, "\n") |
| 152 | } |
| 153 | |
| 154 | // WaitUp wacht op /healthz. |
| 155 | func (g *BlueGit) WaitUp(d time.Duration) { |
| 156 | g.t.Helper() |
| 157 | deadline := time.Now().Add(d) |
| 158 | for time.Now().Before(deadline) { |
| 159 | res, err := http.Get(g.URL + "/healthz") |
| 160 | if err == nil { |
| 161 | res.Body.Close() |
| 162 | if res.StatusCode == 200 { |
| 163 | return |
| 164 | } |
| 165 | } |
| 166 | time.Sleep(100 * time.Millisecond) |
| 167 | } |
| 168 | g.t.Fatalf("BlueGit %s kwam niet op:\n%s", g.URL, tail(g.log.String(), 40)) |
| 169 | } |
| 170 | |
| 171 | // API doet een verzoek met het token. path begint na /api/v1. |
| 172 | func (g *BlueGit) API(method, path string, in any) (int, []byte) { |
| 173 | g.t.Helper() |
| 174 | return g.APIWith(g.Token, method, path, in) |
| 175 | } |
| 176 | |
| 177 | // APIWith gebruikt een ander token. |
| 178 | func (g *BlueGit) APIWith(tok, method, path string, in any) (int, []byte) { |
| 179 | g.t.Helper() |
| 180 | var body io.Reader |
| 181 | if in != nil { |
| 182 | b, _ := json.Marshal(in) |
| 183 | body = bytes.NewReader(b) |
| 184 | } |
| 185 | req, _ := http.NewRequest(method, g.URL+"/api/v1"+path, body) |
| 186 | req.Header.Set("Authorization", "Bearer "+tok) |
| 187 | if in != nil { |
| 188 | req.Header.Set("Content-Type", "application/json") |
| 189 | } |
| 190 | res, err := http.DefaultClient.Do(req) |
| 191 | if err != nil { |
| 192 | g.t.Fatalf("%s %s: %v", method, path, err) |
| 193 | } |
| 194 | defer res.Body.Close() |
| 195 | b, _ := io.ReadAll(res.Body) |
| 196 | return res.StatusCode, b |
| 197 | } |
| 198 | |
| 199 | // MustAPI faalt bij een fout. |
| 200 | func (g *BlueGit) MustAPI(method, path string, in any) []byte { |
| 201 | g.t.Helper() |
| 202 | code, b := g.API(method, path, in) |
| 203 | if code >= 300 { |
| 204 | g.t.Fatalf("%s %s: %d %s", method, path, code, b) |
| 205 | } |
| 206 | return b |
| 207 | } |
| 208 | |
| 209 | // AddServer voegt een backup server toe in BlueGit en geeft id en key. |
| 210 | func (g *BlueGit) AddServer(name string) (string, string) { |
| 211 | g.t.Helper() |
| 212 | var out struct{ ID, Key string } |
| 213 | _ = json.Unmarshal(g.MustAPI("POST", "/admin/backup/servers", map[string]any{"name": name}), &out) |
| 214 | if !strings.HasPrefix(out.Key, "bgbk_") || out.ID == "" { |
| 215 | g.t.Fatalf("server %+v", out) |
| 216 | } |
| 217 | return out.ID, out.Key |
| 218 | } |
| 219 | |
| 220 | // NewKey voegt een backup server toe en geeft zijn Backupserver-Key. |
| 221 | func (g *BlueGit) NewKey() string { |
| 222 | g.t.Helper() |
| 223 | id, key := g.AddServer("test") |
| 224 | g.Server = id |
| 225 | return key |
| 226 | } |
| 227 | |
| 228 | // RotateKey geeft de laatst toegevoegde backup server een nieuwe key. |
| 229 | func (g *BlueGit) RotateKey() string { |
| 230 | g.t.Helper() |
| 231 | var out struct{ Key string } |
| 232 | _ = json.Unmarshal(g.MustAPI("POST", "/admin/backup/servers/"+g.Server+"/key", nil), &out) |
| 233 | return out.Key |
| 234 | } |
| 235 | |
| 236 | // Action voert een actie van backup server id uit via BlueGit. |
| 237 | func (g *BlueGit) Action(id, name string, in any) (int, []byte) { |
| 238 | g.t.Helper() |
| 239 | return g.API("POST", "/admin/backup/servers/"+id+"/actions/"+name, in) |
| 240 | } |
| 241 | |
| 242 | // Project maakt een project met één commit. |
| 243 | func (g *BlueGit) Project(name, file, content string) { |
| 244 | g.t.Helper() |
| 245 | g.MustAPI("POST", "/projects", map[string]any{"name": name, "visibility": "private"}) |
| 246 | g.Commit(strings.ToLower(name), file, content, "create") |
| 247 | } |
| 248 | |
| 249 | // Commit schrijft één bestand op master. |
| 250 | func (g *BlueGit) Commit(repo, file, content, action string) { |
| 251 | g.t.Helper() |
| 252 | g.MustAPI("POST", "/projects/tim/"+repo+"/repository/commits", map[string]any{ |
| 253 | "branch": "master", "commit_message": "zet " + file, |
| 254 | "actions": []map[string]string{{"action": action, "file_path": file, "content": content}}, |
| 255 | }) |
| 256 | } |
| 257 | |
| 258 | // File leest een bestand van master, of geeft de HTTP-code. |
| 259 | func (g *BlueGit) File(repo, file string) (int, string) { |
| 260 | g.t.Helper() |
| 261 | req, _ := http.NewRequest("GET", g.URL+"/api/v1/projects/tim/"+repo+"/repository/files/"+file+"?ref=master", nil) |
| 262 | req.Header.Set("Authorization", "Bearer "+g.Token) |
| 263 | res, err := http.DefaultClient.Do(req) |
| 264 | if err != nil { |
| 265 | g.t.Fatal(err) |
| 266 | } |
| 267 | defer res.Body.Close() |
| 268 | b, _ := io.ReadAll(res.Body) |
| 269 | return res.StatusCode, string(b) |
| 270 | } |
| 271 | |
| 272 | // Issue maakt een issue. |
| 273 | func (g *BlueGit) Issue(repo, title string) { |
| 274 | g.t.Helper() |
| 275 | g.MustAPI("POST", "/repos/tim/"+repo+"/issues", map[string]any{"title": title}) |
| 276 | } |
| 277 | |
| 278 | // IssueTitles geeft alle issuetitels. |
| 279 | func (g *BlueGit) IssueTitles(repo string) []string { |
| 280 | g.t.Helper() |
| 281 | code, b := g.API("GET", "/repos/tim/"+repo+"/issues?state=all", nil) |
| 282 | if code != 200 { |
| 283 | g.t.Fatalf("issues: %d %s", code, b) |
| 284 | } |
| 285 | var list struct { |
| 286 | Issues []struct{ Title string } `json:"issues"` |
| 287 | } |
| 288 | _ = json.Unmarshal(b, &list) |
| 289 | var out []string |
| 290 | for _, i := range list.Issues { |
| 291 | out = append(out, i.Title) |
| 292 | } |
| 293 | return out |
| 294 | } |
| 295 | |
| 296 | // ProjectCode is de HTTP-code van GET project. |
| 297 | func (g *BlueGit) ProjectCode(repo string) int { |
| 298 | g.t.Helper() |
| 299 | code, _ := g.API("GET", "/projects/tim/"+repo, nil) |
| 300 | return code |
| 301 | } |
| 302 | |
| 303 | // BackupStatus is wat BlueGit over een backup server weet. |
| 304 | type BackupStatus struct { |
| 305 | InstanceID string `json:"instance_id"` |
| 306 | HasKey bool `json:"-"` |
| 307 | Name string `json:"name"` |
| 308 | Connected bool `json:"connected"` |
| 309 | ServerURL string `json:"server_url"` |
| 310 | Archives []struct { |
| 311 | Name string `json:"name"` |
| 312 | Size int64 `json:"size"` |
| 313 | } `json:"archives"` |
| 314 | Config struct { |
| 315 | Fields []struct { |
| 316 | Key string `json:"key"` |
| 317 | Value string `json:"value"` |
| 318 | } `json:"fields"` |
| 319 | } `json:"config"` |
| 320 | RemoteError string `json:"remote_error"` |
| 321 | } |
| 322 | |
| 323 | // BackupStatus geeft de stand van de laatst toegevoegde backup server. |
| 324 | func (g *BlueGit) BackupStatus() BackupStatus { return g.ServerStatus(g.Server) } |
| 325 | |
| 326 | // ServerStatus geeft de stand van backup server id ("" = de eerste). |
| 327 | func (g *BlueGit) ServerStatus(id string) BackupStatus { |
| 328 | g.t.Helper() |
| 329 | var list struct { |
| 330 | InstanceID string `json:"instance_id"` |
| 331 | Servers []struct { |
| 332 | ID string `json:"id"` |
| 333 | } `json:"servers"` |
| 334 | } |
| 335 | _ = json.Unmarshal(g.MustAPI("GET", "/admin/backup", nil), &list) |
| 336 | st := BackupStatus{InstanceID: list.InstanceID} |
| 337 | if id == "" && len(list.Servers) > 0 { |
| 338 | id = list.Servers[0].ID |
| 339 | } |
| 340 | found := false |
| 341 | for _, s := range list.Servers { |
| 342 | found = found || s.ID == id |
| 343 | } |
| 344 | if !found { |
| 345 | return st |
| 346 | } |
| 347 | _ = json.Unmarshal(g.MustAPI("GET", "/admin/backup/servers/"+id, nil), &st) |
| 348 | st.HasKey = true |
| 349 | st.InstanceID = list.InstanceID |
| 350 | return st |
| 351 | } |
| 352 | |
| 353 | // Session geeft een HTTP-client met sessiecookie (dev-login). |
| 354 | func (g *BlueGit) Session() *http.Client { |
| 355 | g.t.Helper() |
| 356 | jar, _ := cookiejar.New(nil) |
| 357 | hc := &http.Client{Jar: jar} |
| 358 | res, err := hc.Get(g.URL + "/login") |
| 359 | if err != nil { |
| 360 | g.t.Fatal(err) |
| 361 | } |
| 362 | res.Body.Close() |
| 363 | return hc |
| 364 | } |
| 365 | |
| 366 | // Git draait git met het token als wachtwoord. |
| 367 | func (g *BlueGit) Git(dir string, args ...string) string { |
| 368 | g.t.Helper() |
| 369 | cmd := exec.Command("git", args...) |
| 370 | cmd.Dir = dir |
| 371 | cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_AUTHOR_NAME=it", "GIT_AUTHOR_EMAIL=it@example.invalid", "GIT_COMMITTER_NAME=it", "GIT_COMMITTER_EMAIL=it@example.invalid") |
| 372 | out, err := cmd.CombinedOutput() |
| 373 | if err != nil { |
| 374 | g.t.Fatalf("git %v: %v\n%s", args, err, out) |
| 375 | } |
| 376 | return string(out) |
| 377 | } |
| 378 | |
| 379 | // RemoteURL is de clone-URL met token. |
| 380 | func (g *BlueGit) RemoteURL(repo string) string { |
| 381 | u, _ := url.Parse(g.URL) |
| 382 | u.User = url.UserPassword("tim", g.Token) |
| 383 | u.Path = "/tim/" + repo + ".git" |
| 384 | return u.String() |
| 385 | } |
| 386 | |
| 387 | // Backup is één backup server in dit proces. |
| 388 | type Backup struct { |
| 389 | t *testing.T |
| 390 | Srv *server.Server |
| 391 | URL string |
| 392 | Dir string |
| 393 | Cfg config.Config |
| 394 | } |
| 395 | |
| 396 | func startBackup(t *testing.T, mod func(*config.Config)) *Backup { |
| 397 | t.Helper() |
| 398 | port := freePort(t) |
| 399 | b := &Backup{t: t, URL: fmt.Sprintf("http://127.0.0.1:%d", port), Dir: t.TempDir()} |
| 400 | cfg := config.Config{Listen: fmt.Sprintf("127.0.0.1:%d", port), PublicURL: b.URL, DataDir: b.Dir} |
| 401 | // Snel genoeg voor tests; de standaardwaarden staan in config.Defaults. |
| 402 | cfg.MinSyncInterval.Duration = 50 * time.Millisecond |
| 403 | cfg.ArchiveInterval.Duration = time.Hour |
| 404 | cfg.FullSyncInterval.Duration = time.Hour |
| 405 | cfg.RateLimitKiB = -1 |
| 406 | cfg.MinFreeMB = 1 |
| 407 | if mod != nil { |
| 408 | mod(&cfg) |
| 409 | } |
| 410 | cfg.Defaults() |
| 411 | b.Cfg = cfg |
| 412 | srv, err := server.New(cfg, slog.New(slog.NewTextHandler(&testWriter{t}, &slog.HandlerOptions{Level: slog.LevelWarn}))) |
| 413 | if err != nil { |
| 414 | t.Fatal(err) |
| 415 | } |
| 416 | b.Srv = srv |
| 417 | ln, err := net.Listen("tcp", cfg.Listen) |
| 418 | if err != nil { |
| 419 | t.Fatal(err) |
| 420 | } |
| 421 | hs := &http.Server{Handler: srv.Handler()} |
| 422 | ctx, cancel := context.WithCancel(context.Background()) |
| 423 | go func() { _ = hs.Serve(ln) }() |
| 424 | srv.Run(ctx) |
| 425 | t.Cleanup(func() { |
| 426 | cancel() |
| 427 | _ = hs.Close() |
| 428 | }) |
| 429 | return b |
| 430 | } |
| 431 | |
| 432 | type testWriter struct{ t *testing.T } |
| 433 | |
| 434 | func (w *testWriter) Write(p []byte) (int, error) { |
| 435 | w.t.Log(strings.TrimRight(string(p), "\n")) |
| 436 | return len(p), nil |
| 437 | } |
| 438 | |
| 439 | // Admin doet een verzoek op de lokale beheer-API. |
| 440 | func (b *Backup) Admin(method, path string, in any) (int, []byte) { |
| 441 | b.t.Helper() |
| 442 | var body io.Reader |
| 443 | if in != nil { |
| 444 | raw, _ := json.Marshal(in) |
| 445 | body = bytes.NewReader(raw) |
| 446 | } |
| 447 | req, _ := http.NewRequest(method, b.URL+"/api/v1/admin"+path, body) |
| 448 | req.Header.Set("Authorization", "Bearer "+b.Srv.AdminToken()) |
| 449 | res, err := http.DefaultClient.Do(req) |
| 450 | if err != nil { |
| 451 | b.t.Fatal(err) |
| 452 | } |
| 453 | defer res.Body.Close() |
| 454 | raw, _ := io.ReadAll(res.Body) |
| 455 | return res.StatusCode, raw |
| 456 | } |
| 457 | |
| 458 | // Connect koppelt aan g zonder conflictkeuze; faalt bij een fout. |
| 459 | func (b *Backup) Connect(g *BlueGit, key string) { |
| 460 | b.t.Helper() |
| 461 | code, raw := b.Admin("POST", "/connect", server.ConnectRequest{URL: g.URL, Key: key}) |
| 462 | if code != 200 { |
| 463 | b.t.Fatalf("connect: %d %s", code, raw) |
| 464 | } |
| 465 | } |
| 466 | |
| 467 | // Sync synchroniseert nu. |
| 468 | func (b *Backup) Sync() { |
| 469 | b.t.Helper() |
| 470 | code, raw := b.Admin("POST", "/sync", nil) |
| 471 | if code != 200 { |
| 472 | b.t.Fatalf("sync: %d %s", code, raw) |
| 473 | } |
| 474 | } |
| 475 | |
| 476 | // Archive maakt nu een archief en geeft de naam. |
| 477 | func (b *Backup) Archive() string { |
| 478 | b.t.Helper() |
| 479 | code, raw := b.Admin("POST", "/archive", nil) |
| 480 | if code != 201 { |
| 481 | b.t.Fatalf("archive: %d %s", code, raw) |
| 482 | } |
| 483 | var a struct{ Name string } |
| 484 | _ = json.Unmarshal(raw, &a) |
| 485 | return a.Name |
| 486 | } |
| 487 | |
| 488 | // Mirror is het pad van een logisch bestand in de kopie. |
| 489 | func (b *Backup) Mirror(logical string) string { |
| 490 | return filepath.Join(b.Dir, "mirror", filepath.FromSlash(logical)) |
| 491 | } |
| 492 | |
| 493 | // Archives geeft de archiefnamen, nieuwste eerst. |
| 494 | func (b *Backup) Archives() []string { |
| 495 | b.t.Helper() |
| 496 | _, raw := b.Admin("GET", "/archives", nil) |
| 497 | var out struct { |
| 498 | Archives []struct{ Name string } |
| 499 | } |
| 500 | _ = json.Unmarshal(raw, &out) |
| 501 | var names []string |
| 502 | for _, a := range out.Archives { |
| 503 | names = append(names, a.Name) |
| 504 | } |
| 505 | return names |
| 506 | } |
| 507 | |
| 508 | // Status leest de status. |
| 509 | func (b *Backup) Status() server.Status { |
| 510 | b.t.Helper() |
| 511 | _, raw := b.Admin("GET", "/status", nil) |
| 512 | var st server.Status |
| 513 | _ = json.Unmarshal(raw, &st) |
| 514 | return st |
| 515 | } |
| 516 | |
| 517 | // Eventually herhaalt f tot hij true geeft. |
| 518 | func Eventually(t *testing.T, d time.Duration, what string, f func() bool) { |
| 519 | t.Helper() |
| 520 | deadline := time.Now().Add(d) |
| 521 | for time.Now().Before(deadline) { |
| 522 | if f() { |
| 523 | return |
| 524 | } |
| 525 | time.Sleep(50 * time.Millisecond) |
| 526 | } |
| 527 | t.Fatalf("na %v nog steeds niet: %s", d, what) |
| 528 | } |
| 529 | |
| 530 | // gitLog geeft de onderwerpen van een bare repo in de kopie. |
| 531 | func gitLog(t *testing.T, gitDir string) string { |
| 532 | t.Helper() |
| 533 | out, err := exec.Command("git", "--git-dir", gitDir, "log", "--format=%s", "master").CombinedOutput() |
| 534 | if err != nil { |
| 535 | return "FOUT: " + string(out) |
| 536 | } |
| 537 | return string(out) |
| 538 | } |