| 1 | package backup |
| 2 | |
| 3 | import ( |
| 4 | "archive/tar" |
| 5 | "compress/gzip" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "io" |
| 12 | "io/fs" |
| 13 | "os" |
| 14 | "path" |
| 15 | "path/filepath" |
| 16 | "sort" |
| 17 | "strings" |
| 18 | "sync" |
| 19 | "time" |
| 20 | ) |
| 21 | |
| 22 | // Wat er in een backup zit. Logische paden beginnen met state/ of data/. |
| 23 | // |
| 24 | // state/bluegit.db consistente kopie (VACUUM INTO), niet het live bestand |
| 25 | // state/secret.key nodig om de geheimen in de database te lezen |
| 26 | // state/ssh/… SSH-hostkey |
| 27 | // state/logos/… project- en groepslogo's |
| 28 | // state/avatars/… geüploade avatars |
| 29 | // data/repos/… bare repositories |
| 30 | // data/releases/… gepubliceerde release-assets (zonder .incoming) |
| 31 | // data/artifacts/… CI-artifacts |
| 32 | // |
| 33 | // Niet mee: storage_path, cache.json, backup.json en instance_id (horen bij |
| 34 | // de installatie), data/work (tijdelijke werkmappen), lock-bestanden en |
| 35 | // symlinks. |
| 36 | var ( |
| 37 | stateFiles = []string{"bluegit.db", "secret.key"} |
| 38 | stateDirs = []string{"ssh", "logos", "avatars"} |
| 39 | dataDirs = []string{"repos", "releases", "artifacts"} |
| 40 | ) |
| 41 | |
| 42 | // DBPath is het logische pad van de databasekopie. |
| 43 | const DBPath = "state/bluegit.db" |
| 44 | |
| 45 | // Entry is één bestand in het manifest. |
| 46 | type Entry struct { |
| 47 | Path string `json:"path"` |
| 48 | Size int64 `json:"size"` |
| 49 | MTime int64 `json:"mtime"` // Unix-milliseconden |
| 50 | Mode uint32 `json:"mode"` |
| 51 | SHA256 string `json:"sha256,omitempty"` |
| 52 | } |
| 53 | |
| 54 | // Manifest is de volledige lijst op één moment. |
| 55 | type Manifest struct { |
| 56 | Instance string `json:"instance_id"` |
| 57 | CreatedAt time.Time `json:"created_at"` |
| 58 | Files []Entry `json:"files"` |
| 59 | } |
| 60 | |
| 61 | // Tree kent de twee mappen van een installatie. |
| 62 | type Tree struct { |
| 63 | StateDir string |
| 64 | DataDir string // leeg als er geen datadisk is |
| 65 | // Snapshot schrijft een consistente kopie van de database naar pad. |
| 66 | Snapshot func(dst string) error |
| 67 | // Conn is de koppeling waarvoor de kopie is. Elke backup server krijgt |
| 68 | // een eigen databasekopie, zodat twee servers die tegelijk een |
| 69 | // manifest ophalen elkaars kopie niet vervangen. |
| 70 | Conn string |
| 71 | } |
| 72 | |
| 73 | // snapshotMu zorgt dat er maar één databasekopie tegelijk gemaakt wordt. |
| 74 | var snapshotMu sync.Mutex |
| 75 | |
| 76 | func (t Tree) snapshotPath() string { |
| 77 | name := "snapshot.db" |
| 78 | if t.Conn != "" { |
| 79 | name = "snapshot-" + safeName(t.Conn) + ".db" |
| 80 | } |
| 81 | return filepath.Join(t.StateDir, ".backup", name) |
| 82 | } |
| 83 | |
| 84 | // RemoveSnapshot ruimt de databasekopie van een koppeling op. |
| 85 | func (t Tree) RemoveSnapshot() { _ = os.Remove(t.snapshotPath()) } |
| 86 | |
| 87 | func safeName(s string) string { |
| 88 | var b strings.Builder |
| 89 | for _, r := range s { |
| 90 | if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' { |
| 91 | b.WriteRune(r) |
| 92 | } |
| 93 | } |
| 94 | return b.String() |
| 95 | } |
| 96 | |
| 97 | // Build maakt een nieuwe databasekopie en loopt alle bestanden af. |
| 98 | func (t Tree) Build(instance string) (Manifest, error) { |
| 99 | m := Manifest{Instance: instance, CreatedAt: time.Now().UTC(), Files: []Entry{}} |
| 100 | if t.Snapshot != nil { |
| 101 | snap := t.snapshotPath() |
| 102 | if err := os.MkdirAll(filepath.Dir(snap), 0o700); err != nil { |
| 103 | return m, err |
| 104 | } |
| 105 | tmp := snap + ".new" |
| 106 | snapshotMu.Lock() |
| 107 | _ = os.Remove(tmp) |
| 108 | err := t.Snapshot(tmp) |
| 109 | if err == nil { |
| 110 | err = os.Rename(tmp, snap) |
| 111 | } else { |
| 112 | err = fmt.Errorf("databasekopie: %w", err) |
| 113 | } |
| 114 | snapshotMu.Unlock() |
| 115 | if err != nil { |
| 116 | return m, err |
| 117 | } |
| 118 | e, err := entryOf(DBPath, snap, true) |
| 119 | if err != nil { |
| 120 | return m, err |
| 121 | } |
| 122 | m.Files = append(m.Files, e) |
| 123 | } |
| 124 | for _, f := range stateFiles[1:] { |
| 125 | p := filepath.Join(t.StateDir, f) |
| 126 | if e, err := entryOf("state/"+f, p, false); err == nil { |
| 127 | m.Files = append(m.Files, e) |
| 128 | } |
| 129 | } |
| 130 | for _, d := range stateDirs { |
| 131 | if err := walk(&m, "state/"+d, filepath.Join(t.StateDir, d)); err != nil { |
| 132 | return m, err |
| 133 | } |
| 134 | } |
| 135 | if t.DataDir != "" { |
| 136 | for _, d := range dataDirs { |
| 137 | if err := walk(&m, "data/"+d, filepath.Join(t.DataDir, d)); err != nil { |
| 138 | return m, err |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | sort.Slice(m.Files, func(i, j int) bool { return m.Files[i].Path < m.Files[j].Path }) |
| 143 | return m, nil |
| 144 | } |
| 145 | |
| 146 | func entryOf(logical, real string, hash bool) (Entry, error) { |
| 147 | fi, err := os.Lstat(real) |
| 148 | if err != nil { |
| 149 | return Entry{}, err |
| 150 | } |
| 151 | if !fi.Mode().IsRegular() { |
| 152 | return Entry{}, errors.New("geen gewoon bestand") |
| 153 | } |
| 154 | e := Entry{Path: logical, Size: fi.Size(), MTime: fi.ModTime().UnixMilli(), Mode: uint32(fi.Mode().Perm())} |
| 155 | if hash { |
| 156 | f, err := os.Open(real) |
| 157 | if err != nil { |
| 158 | return e, err |
| 159 | } |
| 160 | defer f.Close() |
| 161 | h := sha256.New() |
| 162 | if _, err := io.Copy(h, f); err != nil { |
| 163 | return e, err |
| 164 | } |
| 165 | e.SHA256 = hex.EncodeToString(h.Sum(nil)) |
| 166 | } |
| 167 | return e, nil |
| 168 | } |
| 169 | |
| 170 | func skipName(name string) bool { |
| 171 | return strings.HasSuffix(name, ".lock") || strings.HasPrefix(name, "tmp_") || name == ".incoming" |
| 172 | } |
| 173 | |
| 174 | func walk(m *Manifest, logical, root string) error { |
| 175 | err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { |
| 176 | if err != nil { |
| 177 | if errors.Is(err, fs.ErrNotExist) { |
| 178 | return nil |
| 179 | } |
| 180 | // Een map die tijdens het lopen verdwijnt (git gc) slaan we over. |
| 181 | if d != nil && d.IsDir() { |
| 182 | return fs.SkipDir |
| 183 | } |
| 184 | return nil |
| 185 | } |
| 186 | if p != root && skipName(d.Name()) { |
| 187 | if d.IsDir() { |
| 188 | return fs.SkipDir |
| 189 | } |
| 190 | return nil |
| 191 | } |
| 192 | if d.IsDir() || d.Type()&fs.ModeSymlink != 0 || !d.Type().IsRegular() { |
| 193 | return nil |
| 194 | } |
| 195 | rel, err := filepath.Rel(root, p) |
| 196 | if err != nil { |
| 197 | return nil |
| 198 | } |
| 199 | if e, err := entryOf(logical+"/"+filepath.ToSlash(rel), p, false); err == nil { |
| 200 | m.Files = append(m.Files, e) |
| 201 | } |
| 202 | return nil |
| 203 | }) |
| 204 | if errors.Is(err, fs.ErrNotExist) { |
| 205 | return nil |
| 206 | } |
| 207 | return err |
| 208 | } |
| 209 | |
| 210 | // CleanLogical controleert een logisch pad en geeft het genormaliseerd terug. |
| 211 | func CleanLogical(p string) (string, bool) { |
| 212 | if p == "" || strings.Contains(p, "\\") || strings.ContainsRune(p, 0) || strings.HasPrefix(p, "/") { |
| 213 | return "", false |
| 214 | } |
| 215 | c := path.Clean(p) |
| 216 | if c != p || c == "." || strings.HasPrefix(c, "../") || strings.Contains(c, "/../") { |
| 217 | return "", false |
| 218 | } |
| 219 | parts := strings.Split(c, "/") |
| 220 | for _, part := range parts { |
| 221 | if part == ".." || part == "." || part == "" { |
| 222 | return "", false |
| 223 | } |
| 224 | } |
| 225 | if len(parts) < 2 { |
| 226 | return "", false |
| 227 | } |
| 228 | switch parts[0] { |
| 229 | case "state": |
| 230 | if len(parts) == 2 && contains(stateFiles, parts[1]) { |
| 231 | return c, true |
| 232 | } |
| 233 | if len(parts) >= 3 && contains(stateDirs, parts[1]) { |
| 234 | return c, true |
| 235 | } |
| 236 | case "data": |
| 237 | if len(parts) >= 3 && contains(dataDirs, parts[1]) && parts[2] != ".incoming" { |
| 238 | return c, true |
| 239 | } |
| 240 | } |
| 241 | return "", false |
| 242 | } |
| 243 | |
| 244 | func contains(list []string, v string) bool { |
| 245 | for _, x := range list { |
| 246 | if x == v { |
| 247 | return true |
| 248 | } |
| 249 | } |
| 250 | return false |
| 251 | } |
| 252 | |
| 253 | // Resolve geeft het echte bestand bij een logisch pad. De database komt |
| 254 | // uit de laatste kopie van Build. |
| 255 | func (t Tree) Resolve(logical string) (string, error) { |
| 256 | c, ok := CleanLogical(logical) |
| 257 | if !ok { |
| 258 | return "", errors.New("ongeldig pad") |
| 259 | } |
| 260 | var real string |
| 261 | switch { |
| 262 | case c == DBPath: |
| 263 | real = t.snapshotPath() |
| 264 | case strings.HasPrefix(c, "state/"): |
| 265 | real = filepath.Join(t.StateDir, filepath.FromSlash(strings.TrimPrefix(c, "state/"))) |
| 266 | default: |
| 267 | if t.DataDir == "" { |
| 268 | return "", errors.New("geen datadisk") |
| 269 | } |
| 270 | real = filepath.Join(t.DataDir, filepath.FromSlash(strings.TrimPrefix(c, "data/"))) |
| 271 | } |
| 272 | fi, err := os.Lstat(real) |
| 273 | if err != nil { |
| 274 | return "", err |
| 275 | } |
| 276 | if !fi.Mode().IsRegular() { |
| 277 | return "", errors.New("geen gewoon bestand") |
| 278 | } |
| 279 | return real, nil |
| 280 | } |
| 281 | |
| 282 | // ---------- terugzetten ---------- |
| 283 | |
| 284 | const restoreDir = ".restore" |
| 285 | |
| 286 | type pending struct { |
| 287 | DataDir string `json:"data_dir"` |
| 288 | At time.Time `json:"at"` |
| 289 | Files int `json:"files"` |
| 290 | } |
| 291 | |
| 292 | // Stage pakt een tar.gz met logische paden uit naast de live mappen. Er |
| 293 | // verandert nog niets aan de live gegevens; dat doet ApplyPending bij de |
| 294 | // volgende start. De database en secret.key moeten in het archief zitten. |
| 295 | func (t Tree) Stage(r io.Reader) (int, error) { |
| 296 | stateStage := filepath.Join(t.StateDir, restoreDir) |
| 297 | _ = os.RemoveAll(stateStage) |
| 298 | var dataStage string |
| 299 | if t.DataDir != "" { |
| 300 | dataStage = filepath.Join(t.DataDir, restoreDir) |
| 301 | _ = os.RemoveAll(dataStage) |
| 302 | } |
| 303 | fail := func(err error) (int, error) { |
| 304 | _ = os.RemoveAll(stateStage) |
| 305 | if dataStage != "" { |
| 306 | _ = os.RemoveAll(dataStage) |
| 307 | } |
| 308 | return 0, err |
| 309 | } |
| 310 | gz, err := gzip.NewReader(r) |
| 311 | if err != nil { |
| 312 | return fail(fmt.Errorf("geen gzip: %w", err)) |
| 313 | } |
| 314 | tr := tar.NewReader(gz) |
| 315 | n := 0 |
| 316 | seenDB, seenKey := false, false |
| 317 | for { |
| 318 | h, err := tr.Next() |
| 319 | if err == io.EOF { |
| 320 | break |
| 321 | } |
| 322 | if err != nil { |
| 323 | return fail(err) |
| 324 | } |
| 325 | if h.Typeflag == tar.TypeDir { |
| 326 | continue |
| 327 | } |
| 328 | if h.Typeflag != tar.TypeReg { |
| 329 | return fail(fmt.Errorf("%s: alleen gewone bestanden", h.Name)) |
| 330 | } |
| 331 | c, ok := CleanLogical(strings.TrimPrefix(h.Name, "./")) |
| 332 | if !ok { |
| 333 | return fail(fmt.Errorf("ongeldig pad in archief: %q", h.Name)) |
| 334 | } |
| 335 | var dst string |
| 336 | if strings.HasPrefix(c, "state/") { |
| 337 | dst = filepath.Join(stateStage, filepath.FromSlash(c)) |
| 338 | } else { |
| 339 | if dataStage == "" { |
| 340 | return fail(errors.New("het archief bevat repositories, maar deze BlueGit heeft nog geen datadisk")) |
| 341 | } |
| 342 | dst = filepath.Join(dataStage, filepath.FromSlash(c)) |
| 343 | } |
| 344 | seenDB = seenDB || c == DBPath |
| 345 | seenKey = seenKey || c == "state/secret.key" |
| 346 | if err := os.MkdirAll(filepath.Dir(dst), 0o750); err != nil { |
| 347 | return fail(err) |
| 348 | } |
| 349 | mode := os.FileMode(h.Mode).Perm() & 0o755 |
| 350 | if mode == 0 { |
| 351 | mode = 0o640 |
| 352 | } |
| 353 | f, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode|0o600) |
| 354 | if err != nil { |
| 355 | return fail(err) |
| 356 | } |
| 357 | if _, err := io.Copy(f, tr); err != nil { |
| 358 | f.Close() |
| 359 | return fail(err) |
| 360 | } |
| 361 | if err := f.Close(); err != nil { |
| 362 | return fail(err) |
| 363 | } |
| 364 | _ = os.Chtimes(dst, h.ModTime, h.ModTime) |
| 365 | n++ |
| 366 | } |
| 367 | if !seenDB || !seenKey { |
| 368 | return fail(errors.New("het archief mist state/bluegit.db of state/secret.key")) |
| 369 | } |
| 370 | b, _ := json.Marshal(pending{DataDir: t.DataDir, At: time.Now().UTC(), Files: n}) |
| 371 | if err := os.WriteFile(filepath.Join(stateStage, "READY"), b, 0o600); err != nil { |
| 372 | return fail(err) |
| 373 | } |
| 374 | return n, nil |
| 375 | } |
| 376 | |
| 377 | // HasPending zegt of er een uitgepakte backup klaarstaat. |
| 378 | func HasPending(stateDir string) bool { |
| 379 | _, err := os.Stat(filepath.Join(stateDir, restoreDir, "READY")) |
| 380 | return err == nil |
| 381 | } |
| 382 | |
| 383 | // ApplyPending zet een klaarstaande backup op zijn plek. Aanroepen bij de |
| 384 | // start, vóór de database open gaat. Geen READY: niets te doen. |
| 385 | func ApplyPending(stateDir string) (bool, error) { |
| 386 | stage := filepath.Join(stateDir, restoreDir) |
| 387 | b, err := os.ReadFile(filepath.Join(stage, "READY")) |
| 388 | if errors.Is(err, os.ErrNotExist) { |
| 389 | return false, nil |
| 390 | } |
| 391 | if err != nil { |
| 392 | return false, err |
| 393 | } |
| 394 | var p pending |
| 395 | if err := json.Unmarshal(b, &p); err != nil { |
| 396 | return false, err |
| 397 | } |
| 398 | for _, f := range []string{"bluegit.db-wal", "bluegit.db-shm"} { |
| 399 | _ = os.Remove(filepath.Join(stateDir, f)) |
| 400 | } |
| 401 | for _, name := range append(append([]string{}, stateFiles...), stateDirs...) { |
| 402 | if err := swap(filepath.Join(stateDir, name), filepath.Join(stage, "state", name)); err != nil { |
| 403 | return false, err |
| 404 | } |
| 405 | } |
| 406 | if p.DataDir != "" { |
| 407 | dstage := filepath.Join(p.DataDir, restoreDir) |
| 408 | for _, name := range dataDirs { |
| 409 | if err := swap(filepath.Join(p.DataDir, name), filepath.Join(dstage, "data", name)); err != nil { |
| 410 | return false, err |
| 411 | } |
| 412 | } |
| 413 | _ = os.RemoveAll(dstage) |
| 414 | } |
| 415 | _ = os.RemoveAll(filepath.Join(stateDir, ".backup")) |
| 416 | return true, os.RemoveAll(stage) |
| 417 | } |
| 418 | |
| 419 | // swap vervangt live door staged. Ontbreekt staged, dan verdwijnt live. |
| 420 | func swap(live, staged string) error { |
| 421 | old := live + ".restore-old" |
| 422 | _ = os.RemoveAll(old) |
| 423 | if _, err := os.Lstat(live); err == nil { |
| 424 | if err := os.Rename(live, old); err != nil { |
| 425 | return err |
| 426 | } |
| 427 | } |
| 428 | if _, err := os.Lstat(staged); err == nil { |
| 429 | if err := os.Rename(staged, live); err != nil { |
| 430 | return err |
| 431 | } |
| 432 | } |
| 433 | return os.RemoveAll(old) |
| 434 | } |