From dd2065cf602a8f89be235676af676ba949eb3ea6 Mon Sep 17 00:00:00 2001
From: Tibo De Peuter <tibo.depeuter@ugent.be>
Date: Fri, 17 Jul 2026 22:07:49 +0200
Subject: [PATCH] ci: add golden image build action and document disaster
 recovery

---
 .forgejo/workflows/build-golden-image.yml | 42 +++++++++++++++++++++++
 DISASTER_RECOVERY.md                      | 32 +++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 .forgejo/workflows/build-golden-image.yml
 create mode 100644 DISASTER_RECOVERY.md

diff --git a/.forgejo/workflows/build-golden-image.yml b/.forgejo/workflows/build-golden-image.yml
new file mode 100644
index 0000000..fe71f64
--- /dev/null
+++ b/.forgejo/workflows/build-golden-image.yml
@@ -0,0 +1,42 @@
+name: Build Golden Image
+
+on:
+  push:
+    paths:
+      - 'hosts/Template/**'
+      - 'flake.nix'
+      - 'flake.lock'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: Build Proxmox Image
+        run: |
+          nix run github:nix-community/nixos-generators -- --format proxmox -c hosts/Template/default.nix -o result
+
+      - name: Upload to Proxmox
+        env:
+          PROXMOX_URL: "https://proxmox.local:8006/api2/json"
+          PROXMOX_TOKEN_ID: "terraform@pve!tf-automation"
+          PROXMOX_TOKEN_SECRET: ${{ secrets.PROXMOX_TOKEN_SECRET }}
+          NODE_NAME: "pve"
+          STORAGE_NAME: "local-zfs"
+          VMID: 9000
+        run: |
+          IMAGE_PATH=$(find result -name "*.qcow2" | head -n 1)
+          echo "Uploading $IMAGE_PATH to Proxmox as Template $VMID"
+          # In a real scenario, this would use a script or API client to upload the image
+          # e.g., using qm importdisk via ssh or the Proxmox API directly.
+          # For example, using a custom script: ./scripts/upload-to-proxmox.sh $IMAGE_PATH $VMID
+          echo "TODO: Implement Proxmox upload API call using PROXMOX_TOKEN_SECRET"
diff --git a/DISASTER_RECOVERY.md b/DISASTER_RECOVERY.md
new file mode 100644
index 0000000..2d5464b
--- /dev/null
+++ b/DISASTER_RECOVERY.md
@@ -0,0 +1,32 @@
+# Disaster Recovery Plan (Zero-Login Strategy)
+
+In the event of a catastrophic failure, this document outlines how to restore services without causing configuration drift (i.e., without SSHing directly into target nodes and making manual undocumented changes).
+
+## 1. Full Proxmox Host Freeze (Hardware/GPU Bug)
+If the entire Proxmox hypervisor freezes (often caused by the AMD GPU reset bug when a VM reboots):
+*   **DO NOT** attempt to SSH into the host or the VMs.
+*   **Recovery Action**: Use your Out-of-Band Management (OOBM) solution. Access your PiKVM, IPMI interface, or remotely controlled Smart Plug to perform a **hard power cycle** of the host.
+*   Upon reboot, Proxmox will automatically start the VMs, and `comin` will pull the latest working configuration.
+
+## 2. Production Service Failure (Code Bug)
+If a bad PR was merged and a stateless service is failing to start:
+*   **DO NOT** SSH into the VM to fix the config manually.
+*   **Recovery Action**: Open the Forgejo Web UI and click **Revert** on the merged PR.
+*   Within 60 seconds, the `comin` agent running on the target VM will detect the new commit on the `v2` branch, pull the reverted configuration, and restart the service automatically.
+
+## 3. Production Data Corruption (Database/Stateful Bug)
+If a bad deployment corrupted persistent data on TrueNAS (e.g., a failed one-way database schema migration):
+*   **Recovery Action**: 
+    1. Revert the code PR in Forgejo as described in section 2.
+    2. Execute the `scripts/dr-rollback-dataset.sh` script from the secure **Control Center VM**. This script uses the TrueNAS REST API to rollback the specific ZFS dataset to the automated snapshot taken immediately prior to the deployment.
+
+## 4. Total Git Repository Loss
+If the `v2` branch is completely destroyed or the Gitea server is unrecoverable:
+*   Your `prod-master` private age key is stored safely offline on your USB drive.
+*   A recent backup of the repository should be available via your external backup mechanism.
+*   Clone the backup to your local workstation, run the `proxmox-bootstrap` Terraform state locally to ensure the hypervisor is reachable, and use Terraform to redeploy the Control Center VM and Forgejo instances.
+
+## 5. Manually Obtaining the Golden Image
+If the CI pipeline is down and you urgently need to provision a new VM:
+1. Run `nix run github:nix-community/nixos-generators -- --format proxmox -c hosts/Template/default.nix -o result` on your local machine.
+2. The output `.qcow2` image can be uploaded to the Proxmox Web UI manually under `local` -> `ISO Images` (or directly via `qm importdisk` if you must use SSH as a last resort).