diff --git a/app/backup.py b/app/backup.py index 3b21a65..25b9460 100644 --- a/app/backup.py +++ b/app/backup.py @@ -163,36 +163,68 @@ def backup_stack(name: str, live_stack: dict, stack_cfg: dict, config: dict) -> entry = {"stack": name, "ts": _iso_now(), "status": "ok", "file": os.path.basename(tar_path), "size": 0, "error": "", "warnings": []} + + if settings.DEV_MODE: + log.info("[dev] simulating backup of %s (%d paths)", name, len(paths)) + _write_placeholder_tar(tar_path, paths, extra_files) + entry["status"] = "simulated" + if os.path.exists(tar_path): + entry["size"] = os.path.getsize(tar_path) + _finish(entry, folder, config) + return entry + + # 1) bring the stack DOWN (client retries + verifies) try: - if settings.DEV_MODE: - log.info("[dev] simulating backup of %s (%d paths)", name, len(paths)) - _write_placeholder_tar(tar_path, paths, extra_files) - entry["status"] = "simulated" - else: - dockhand.client.down(name) - try: - warnings = _write_tar(tar_path, paths, extra_files) - entry["warnings"] = warnings - host_missing = [w for w in warnings] - if paths and len(host_missing) == len(paths): - entry["status"] = "error" - entry["error"] = "all host paths missing" - elif warnings: - entry["status"] = "partial" - finally: - dockhand.client.start(name) # never leave a stack down + dockhand.client.down(name) + except Exception as exc: # noqa: BLE001 + log.exception("down of %s failed", name) + entry["status"] = "error" + entry["error"] = f"down fehlgeschlagen: {exc}" + # Stack is likely still up; don't tar an inconsistent state. Try to + # ensure it's up, then bail. + _safe_start(name, entry) + _finish(entry, folder, config) + return entry + + # 2) TAR — record its result independently of the up step + try: + warnings = _write_tar(tar_path, paths, extra_files) + entry["warnings"] = warnings + if paths and len(warnings) == len(paths): + entry["status"] = "error" + entry["error"] = "alle Host-Pfade fehlen" + elif warnings: + entry["status"] = "partial" if os.path.exists(tar_path): entry["size"] = os.path.getsize(tar_path) except Exception as exc: # noqa: BLE001 - log.exception("backup of %s failed", name) + log.exception("tar of %s failed", name) entry["status"] = "error" - entry["error"] = str(exc) + entry["error"] = f"Tar fehlgeschlagen: {exc}" + # 3) always bring it back UP — a failure here is a SEPARATE flag, it does + # NOT turn a good backup into an error. + _safe_start(name, entry) + _finish(entry, folder, config) + return entry + + +def _safe_start(name: str, entry: dict): + """Bring a stack back up; record an up-failure as a warning, never overwrite + the tar result. The client already retries + verifies status.""" + try: + dockhand.client.start(name) + except Exception as exc: # noqa: BLE001 + log.error("could not bring %s back up: %s", name, exc) + entry["warnings"] = list(entry.get("warnings", [])) + [f"UP fehlgeschlagen: {exc}"] + entry["up_failed"] = True + + +def _finish(entry: dict, folder: str, config: dict): deleted = _apply_retention(folder, config.get("retention_n", 7)) if deleted: - log.info("retention: deleted %d old backups for %s", deleted, name) + log.info("retention: deleted %d old backups for %s", deleted, entry["stack"]) store.update(lambda c: store.add_history(c, entry)) - return entry def backup_custom_paths() -> list: diff --git a/app/dockhand.py b/app/dockhand.py index cdba6e1..acacc3d 100644 --- a/app/dockhand.py +++ b/app/dockhand.py @@ -16,6 +16,7 @@ requests.Session. On 401 we log in once and retry. import json import logging import threading +import time import urllib.parse import requests @@ -158,14 +159,57 @@ class DockhandClient: return data return {} + def status(self, name: str): + """Current status of a stack, or None if unknown/unreachable.""" + try: + for s in self.list_stacks(): + if s.get("name") == name: + return s.get("status") + except Exception as exc: # noqa: BLE001 + log.warning("status check failed for %s: %s", name, exc) + return None + + def _lifecycle(self, name, action, ok_states, *, tolerate=(), body=None, + attempts=5) -> str: + """Run a lifecycle action, then VERIFY via status — with retries. + + Dockhand is known to drop the connection during UP even though the + operation completes server-side, so we always re-check the status + after a (possibly failed) call instead of trusting the response alone. + """ + path = f"/api/stacks/{_q(name)}/{action}" + last = None + for i in range(attempts): + try: + self._run_sse(path, body=body) + except (requests.exceptions.RequestException, DockhandError) as exc: + last = exc + log.warning("%s %s attempt %d/%d: %s", action, name, i + 1, attempts, exc) + st = self.status(name) + if st in ok_states: + return f"{action} ok (status={st})" + if i < attempts - 1: + time.sleep(min(2 ** i, 15)) + st = self.status(name) + if st in ok_states: + return f"{action} ok (status={st})" + if st in tolerate: + log.warning("%s %s ended in tolerated state '%s'", action, name, st) + return f"{action} tolerated (status={st})" + raise DockhandError( + f"{action} {name} failed after {attempts} tries (status={st}); last={last}") + def start(self, name: str) -> str: - return self._run_sse(f"/api/stacks/{_q(name)}/start") + # running = fully up; partial (some containers up) is tolerated, not down. + return self._lifecycle(name, "start", ok_states={"running"}, + tolerate={"partial"}) def stop(self, name: str) -> str: - return self._run_sse(f"/api/stacks/{_q(name)}/stop") + return self._lifecycle(name, "stop", ok_states={"stopped", "created", "exited"}) def down(self, name: str) -> str: - return self._run_sse(f"/api/stacks/{_q(name)}/down", body={"removeVolumes": False}) + return self._lifecycle(name, "down", ok_states={"stopped", "created", "exited"}, + body={"removeVolumes": False}) # ------------------------------------------------------------------ mock diff --git a/docker-compose.yml b/docker-compose.yml index ba21070..b9dea29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - /mnt/cache/appdata:/mnt/cache/appdata:ro - /mnt/docker/appdata:/mnt/docker/appdata:ro - /mnt/user/appdata:/mnt/user/appdata:ro + - /mnt/user/git:/mnt/user/git:ro networks: - nginx