306 lines
12 KiB
Python
306 lines
12 KiB
Python
"""Backup engine: discover stacks, cold-backup each one (down -> tar -> up),
|
|
apply retention, and back up arbitrary custom paths.
|
|
|
|
Format is ALWAYS an uncompressed .tar (never .tar.gz), by requirement.
|
|
Host bind-mount paths are stored with their leading "/" stripped, so a backup
|
|
restores to the original location with: tar xf <file>.tar -C /
|
|
Compose + env are fetched from the Dockhand API and embedded under
|
|
dockhand-meta/<stack>/ (compose.yaml, env) as reference copies.
|
|
|
|
In DEV_MODE the down/up lifecycle and the tarring of real host paths are
|
|
SIMULATED (a tiny placeholder .tar is written); read-only calls (list, inspect,
|
|
compose/env) still run so the UI shows real data.
|
|
"""
|
|
import io
|
|
import logging
|
|
import os
|
|
import stat
|
|
import tarfile
|
|
import threading
|
|
|
|
from . import dockhand, settings, store, timeutil
|
|
|
|
log = logging.getLogger("backup")
|
|
|
|
_RUN_LOCK = threading.Lock()
|
|
|
|
|
|
def _now_stamp() -> str:
|
|
now = timeutil.now()
|
|
return now.strftime("%Y%m%d-%H%M%S") + f"-{now.microsecond // 1000:03d}"
|
|
|
|
|
|
def _iso_now() -> str:
|
|
return timeutil.now().isoformat(timespec="seconds")
|
|
|
|
|
|
# --- discovery ----------------------------------------------------------
|
|
def _live_map():
|
|
return {s["name"]: s for s in dockhand.client.list_stacks()}
|
|
|
|
|
|
def discover(persist: bool = True, live=None) -> dict:
|
|
"""Add any unknown live stacks to the config (enabled, appended)."""
|
|
names = set((live or _live_map()).keys())
|
|
|
|
def mutate(config):
|
|
if not config.get("auto_discover", True):
|
|
return
|
|
known = {s["name"] for s in config["stacks"]}
|
|
nxt = max((s.get("order", 0) for s in config["stacks"]), default=-1) + 1
|
|
for name in sorted(names - known):
|
|
config["stacks"].append(
|
|
{"name": name, "order": nxt, "enabled": True,
|
|
"extra_paths": [], "bind_paths": [], "disabled_binds": []})
|
|
nxt += 1
|
|
|
|
if persist:
|
|
return store.update(mutate)
|
|
config = store.load()
|
|
mutate(config)
|
|
return config
|
|
|
|
|
|
def _cache_bind_paths(name: str, paths: list):
|
|
def mutate(config):
|
|
s = store.get_stack(config, name)
|
|
if s is not None:
|
|
s["bind_paths"] = paths
|
|
store.update(mutate)
|
|
|
|
|
|
# --- tar helpers --------------------------------------------------------
|
|
def _write_tar(tar_path: str, paths: list, extra_files: dict = None) -> list:
|
|
"""Create an uncompressed tar of host paths + in-memory extra files.
|
|
Missing host paths are skipped and returned as warnings."""
|
|
warnings = []
|
|
os.makedirs(os.path.dirname(tar_path), exist_ok=True)
|
|
with tarfile.open(tar_path, "w") as tar: # "w" == uncompressed
|
|
for p in paths:
|
|
if not os.path.exists(p):
|
|
warnings.append(f"missing: {p}")
|
|
log.warning("path missing, skipped: %s", p)
|
|
continue
|
|
mode = os.stat(p).st_mode
|
|
if not (stat.S_ISDIR(mode) or stat.S_ISREG(mode)):
|
|
warnings.append(f"special file skipped: {p}")
|
|
log.warning("special file skipped: %s", p)
|
|
continue
|
|
arcname = p.lstrip("/").lstrip("\\") or os.path.basename(p)
|
|
tar.add(p, arcname=arcname)
|
|
for arcname, content in (extra_files or {}).items():
|
|
data = content.encode() if isinstance(content, str) else content
|
|
info = tarfile.TarInfo(name=arcname)
|
|
info.size = len(data)
|
|
tar.addfile(info, io.BytesIO(data))
|
|
return warnings
|
|
|
|
|
|
def _write_placeholder_tar(tar_path: str, paths: list, extra_files: dict = None):
|
|
os.makedirs(os.path.dirname(tar_path), exist_ok=True)
|
|
body = ("DEV_MODE simulated backup\n\nWould archive host paths:\n"
|
|
+ "\n".join(f" - {p}" for p in paths)
|
|
+ "\n\nPlus embedded files:\n"
|
|
+ "\n".join(f" - {k}" for k in (extra_files or {})) + "\n").encode()
|
|
with tarfile.open(tar_path, "w") as tar:
|
|
info = tarfile.TarInfo(name="DEV_MODE_SIMULATED.txt")
|
|
info.size = len(body)
|
|
tar.addfile(info, io.BytesIO(body))
|
|
|
|
|
|
def _apply_retention(folder: str, keep_n: int) -> int:
|
|
if keep_n <= 0 or not os.path.isdir(folder):
|
|
return 0
|
|
tars = [os.path.join(folder, f) for f in os.listdir(folder) if f.endswith(".tar")]
|
|
tars.sort(key=lambda f: os.path.getmtime(f), reverse=True)
|
|
deleted = 0
|
|
for f in tars[keep_n:]:
|
|
try:
|
|
os.remove(f)
|
|
deleted += 1
|
|
except OSError as exc:
|
|
log.warning("could not delete old backup %s: %s", f, exc)
|
|
return deleted
|
|
|
|
|
|
# --- per-stack backup ---------------------------------------------------
|
|
def _gather(name: str, live_stack: dict, stack_cfg: dict, config: dict):
|
|
"""Return (host_paths, extra_files) for a stack and cache its bind paths."""
|
|
bind = dockhand.bind_paths_for_stack(dockhand.client, live_stack or {})
|
|
_cache_bind_paths(name, bind) # cache ALL detected binds (for the UI toggles)
|
|
|
|
# Drop binds the user switched off — silently, no "missing" warning.
|
|
disabled = set(stack_cfg.get("disabled_binds") or [])
|
|
paths = [b for b in bind if b not in disabled]
|
|
base = config.get("dockhand_stacks_base", "")
|
|
if base:
|
|
paths.append(os.path.join(base, name))
|
|
for p in stack_cfg.get("extra_paths") or []:
|
|
if p:
|
|
paths.append(p)
|
|
# de-dup
|
|
seen, uniq = set(), []
|
|
for p in paths:
|
|
if p not in seen:
|
|
seen.add(p)
|
|
uniq.append(p)
|
|
|
|
extra_files = {}
|
|
compose = dockhand.client.get_compose(name)
|
|
if compose:
|
|
extra_files[f"dockhand-meta/{name}/compose.yaml"] = compose
|
|
env = dockhand.client.get_env_raw(name)
|
|
if env:
|
|
extra_files[f"dockhand-meta/{name}/env"] = env
|
|
return uniq, extra_files
|
|
|
|
|
|
def backup_stack(name: str, live_stack: dict, stack_cfg: dict, config: dict) -> dict:
|
|
"""Cold-backup a single stack: down -> tar -> up. Always brought back up."""
|
|
folder = os.path.join(settings.BACKUP_ROOT, name)
|
|
tar_path = os.path.join(folder, f"{name}_{_now_stamp()}.tar")
|
|
paths, extra_files = _gather(name, live_stack, stack_cfg, config)
|
|
|
|
entry = {"stack": name, "ts": _iso_now(), "status": "ok",
|
|
"file": os.path.basename(tar_path), "size": 0, "error": "", "warnings": []}
|
|
|
|
if settings.DEV_MODE:
|
|
log.info("[dev] simulating backup of %s (%d paths)", name, len(paths))
|
|
_write_placeholder_tar(tar_path, paths, extra_files)
|
|
entry["status"] = "simulated"
|
|
if os.path.exists(tar_path):
|
|
entry["size"] = os.path.getsize(tar_path)
|
|
_finish(entry, folder, config)
|
|
return entry
|
|
|
|
# 1) bring the stack DOWN (client retries + verifies)
|
|
try:
|
|
dockhand.client.down(name)
|
|
except Exception as exc: # noqa: BLE001
|
|
log.exception("down of %s failed", name)
|
|
entry["status"] = "error"
|
|
entry["error"] = f"down fehlgeschlagen: {exc}"
|
|
# Stack is likely still up; don't tar an inconsistent state. Try to
|
|
# ensure it's up, then bail.
|
|
_safe_start(name, entry)
|
|
_finish(entry, folder, config)
|
|
return entry
|
|
|
|
# 2) TAR — record its result independently of the up step
|
|
try:
|
|
warnings = _write_tar(tar_path, paths, extra_files)
|
|
entry["warnings"] = warnings
|
|
if paths and len(warnings) == len(paths):
|
|
entry["status"] = "error"
|
|
entry["error"] = "alle Host-Pfade fehlen"
|
|
elif warnings:
|
|
entry["status"] = "partial"
|
|
if os.path.exists(tar_path):
|
|
entry["size"] = os.path.getsize(tar_path)
|
|
except Exception as exc: # noqa: BLE001
|
|
log.exception("tar of %s failed", name)
|
|
entry["status"] = "error"
|
|
entry["error"] = f"Tar fehlgeschlagen: {exc}"
|
|
|
|
# 3) always bring it back UP — a failure here is a SEPARATE flag, it does
|
|
# NOT turn a good backup into an error.
|
|
_safe_start(name, entry)
|
|
_finish(entry, folder, config)
|
|
return entry
|
|
|
|
|
|
def _safe_start(name: str, entry: dict):
|
|
"""Bring a stack back up; record an up-failure as a warning, never overwrite
|
|
the tar result. The client already retries + verifies status."""
|
|
try:
|
|
dockhand.client.start(name)
|
|
except Exception as exc: # noqa: BLE001
|
|
log.error("could not bring %s back up: %s", name, exc)
|
|
entry["warnings"] = list(entry.get("warnings", [])) + [f"UP fehlgeschlagen: {exc}"]
|
|
entry["up_failed"] = True
|
|
|
|
|
|
def _finish(entry: dict, folder: str, config: dict):
|
|
deleted = _apply_retention(folder, config.get("retention_n", 7))
|
|
if deleted:
|
|
log.info("retention: deleted %d old backups for %s", deleted, entry["stack"])
|
|
store.update(lambda c: store.add_history(c, entry))
|
|
|
|
|
|
def backup_custom_paths() -> list:
|
|
config = store.load()
|
|
results = []
|
|
for item in config.get("custom_paths", []):
|
|
if not item.get("enabled", True):
|
|
continue
|
|
label = item.get("label") or "path"
|
|
path = item.get("path")
|
|
folder = os.path.join(settings.BACKUP_ROOT, "_custom")
|
|
tar_path = os.path.join(folder, f"{label}_{_now_stamp()}.tar")
|
|
entry = {"stack": f"custom:{label}", "ts": _iso_now(), "status": "ok",
|
|
"file": os.path.basename(tar_path), "size": 0, "error": "", "warnings": []}
|
|
try:
|
|
if settings.DEV_MODE:
|
|
_write_placeholder_tar(tar_path, [path])
|
|
entry["status"] = "simulated"
|
|
else:
|
|
warnings = _write_tar(tar_path, [path])
|
|
entry["warnings"] = warnings
|
|
if warnings:
|
|
entry["status"] = "error"
|
|
entry["error"] = "; ".join(warnings)
|
|
if os.path.exists(tar_path):
|
|
entry["size"] = os.path.getsize(tar_path)
|
|
except Exception as exc: # noqa: BLE001
|
|
entry["status"] = "error"
|
|
entry["error"] = str(exc)
|
|
_apply_retention(folder, config.get("retention_n", 7))
|
|
store.update(lambda c, e=entry: store.add_history(c, e))
|
|
results.append(entry)
|
|
return results
|
|
|
|
|
|
# --- run orchestration --------------------------------------------------
|
|
def run_all() -> dict:
|
|
if not _RUN_LOCK.acquire(blocking=False):
|
|
log.warning("backup run already in progress; skipping")
|
|
return {"skipped": True}
|
|
try:
|
|
live = _live_map()
|
|
config = discover(persist=True, live=live)
|
|
results = []
|
|
for stack_cfg in store.stacks_by_order(config):
|
|
if stack_cfg.get("enabled", True):
|
|
name = stack_cfg["name"]
|
|
results.append(backup_stack(name, live.get(name, {}), stack_cfg, config))
|
|
custom = backup_custom_paths()
|
|
return {"stacks": results, "custom": custom}
|
|
finally:
|
|
_RUN_LOCK.release()
|
|
|
|
|
|
def run_one(stack_name: str) -> dict:
|
|
if not _RUN_LOCK.acquire(blocking=False):
|
|
return {"skipped": True}
|
|
try:
|
|
live = _live_map()
|
|
config = store.load()
|
|
stack_cfg = store.get_stack(config, stack_name) or {"name": stack_name, "extra_paths": []}
|
|
return backup_stack(stack_name, live.get(stack_name, {}), stack_cfg, config)
|
|
finally:
|
|
_RUN_LOCK.release()
|
|
|
|
|
|
def refresh_bind_paths() -> dict:
|
|
"""Update the cached bind paths for all known stacks (read-only inspects).
|
|
Used to populate dashboard counts without running a backup."""
|
|
live = _live_map()
|
|
config = store.load()
|
|
counts = {}
|
|
for stack_cfg in config.get("stacks", []):
|
|
name = stack_cfg["name"]
|
|
paths = dockhand.bind_paths_for_stack(dockhand.client, live.get(name, {}))
|
|
counts[name] = len(paths)
|
|
_cache_bind_paths(name, paths)
|
|
return counts
|