.forgejo/workflows/build-golden-image.yml

@@ -0,0 +1,42 @@
+name: Build Golden Image
+
+on:
+  push:
+    paths:
+      - 'hosts/Template/**'
+      - 'flake.nix'
+      - 'flake.lock'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: Build Proxmox Image
+        run: |
+          nix run github:nix-community/nixos-generators -- --format proxmox -c hosts/Template/default.nix -o result
+
+      - name: Upload to Proxmox
+        env:
+          PROXMOX_URL: "https://proxmox.local:8006/api2/json"
+          PROXMOX_TOKEN_ID: "terraform@pve!tf-automation"
+          PROXMOX_TOKEN_SECRET: ${{ secrets.PROXMOX_TOKEN_SECRET }}
+          NODE_NAME: "pve"
+          STORAGE_NAME: "local-zfs"
+          VMID: 9000
+        run: |
+          IMAGE_PATH=$(find result -name "*.qcow2" | head -n 1)
+          echo "Uploading $IMAGE_PATH to Proxmox as Template $VMID"
+          # In a real scenario, this would use a script or API client to upload the image
+          # e.g., using qm importdisk via ssh or the Proxmox API directly.
+          # For example, using a custom script: ./scripts/upload-to-proxmox.sh $IMAGE_PATH $VMID
+          echo "TODO: Implement Proxmox upload API call using PROXMOX_TOKEN_SECRET"

.forgejo/workflows/renovate.yml

@@ -0,0 +1,25 @@
+name: RenovateBot
+
+on:
+  schedule:
+    # Run at 2:00 AM every day
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+
+jobs:
+  renovate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Self-hosted Renovate
+        uses: renovatebot/github-action@v40.1.11
+        env:
+          # Token needs permission to read/write repository contents and pull requests
+          RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
+          # The platform needs to be explicitly defined for Forgejo/Gitea
+          RENOVATE_PLATFORM: "gitea"
+          RENOVATE_ENDPOINT: "https://your-forgejo-instance.local/api/v1"
+          # The repository format: owner/repo
+          RENOVATE_REPOSITORIES: "your-username/nix-config"

.forgejo/workflows/staging.yml

@@ -0,0 +1,76 @@
+name: Dynamic Staging Environment
+
+on:
+  pull_request:
+    types: [opened, synchronize, closed]
+
+jobs:
+  validate-flake:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: Validate Flake
+        run: nix flake check
+
+  manage-staging:
+    runs-on: self-hosted
+    needs: validate-flake
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+
+      - name: Provision Staging Environment (Opened/Sync)
+        if: github.event.action == 'opened' || github.event.action == 'synchronize'
+        env:
+          PROXMOX_VE_ENDPOINT: "https://proxmox.local:8006/"
+          PROXMOX_VE_API_TOKEN: ${{ secrets.PROXMOX_TOKEN_SECRET }}
+          TF_VAR_pr_number: ${{ github.event.pull_request.number }}
+          # VM_ID could be dynamically generated or based on PR number (e.g., 8000 + PR_NUMBER)
+          TF_VAR_vm_id: ${{ format('8{0:03}', github.event.pull_request.number) }}
+          
+          # TrueNAS integration (Secrets would need to be added to Forgejo)
+          TRUENAS_IP: "truenas.local"
+          TRUENAS_API_KEY: ${{ secrets.TRUENAS_API_KEY }}
+          POOL_NAME: "tank"
+          SOURCE_DATASET: "apps/production_data"
+        run: |
+          echo "Cloning datasets..."
+          # ./scripts/truenas-staging-clone.sh
+          
+          echo "Applying Terraform for PR ${{ github.event.pull_request.number }}..."
+          cd terraform/staging-env
+          terraform init
+          terraform apply -auto-approve
+
+      - name: Teardown Staging Environment (Closed)
+        if: github.event.action == 'closed'
+        env:
+          PROXMOX_VE_ENDPOINT: "https://proxmox.local:8006/"
+          PROXMOX_VE_API_TOKEN: ${{ secrets.PROXMOX_TOKEN_SECRET }}
+          TF_VAR_pr_number: ${{ github.event.pull_request.number }}
+          TF_VAR_vm_id: ${{ format('8{0:03}', github.event.pull_request.number) }}
+          
+          # TrueNAS integration
+          TRUENAS_IP: "truenas.local"
+          TRUENAS_API_KEY: ${{ secrets.TRUENAS_API_KEY }}
+          POOL_NAME: "tank"
+          SOURCE_DATASET: "apps/production_data"
+        run: |
+          echo "Destroying Terraform environment for PR ${{ github.event.pull_request.number }}..."
+          cd terraform/staging-env
+          terraform init
+          terraform destroy -auto-approve
+          
+          echo "Tearing down datasets..."
+          # ./scripts/truenas-staging-teardown.sh

.sops.yaml

@@ -1,10 +1,29 @@
+# SOPS configuration for NixOS GitOps Migration (v2)
+# See README.md for key management instructions.
+
 keys:
-  - &tdpeuter_Tibo-NixFatDesk age1fva6s64s884z0q2w7de024sp69ucvqu0pg9shrhhqsn3ewlpjfpsh6md7y
-  - &tdpeuter_Tibo-NixTop age1qzutny0mqpcccqw6myyfntu6wcskruu9ghzvt6r4te7afkqwnguq05ex37
+  # Master Keys (Used for generic/global secrets if needed, but usually we encrypt for specific hosts)
+  - &prod_master age1pq12tgz8e980yvrsvd6c6ct6fa8y8eq0c8hjkdfvhp9k0phsdpjvewrrj42zsj9xhk2h2q6wmzvfnpjcgrm8x2zyzxjwv6urjpe60w89n9c2c0lw4j44qzr5j6ryxnmw8hysvngnf50s7ykrdrvy0aql9ktqqks2zzn48rd0k2mecw2snug23ydc4u4w6vphgusp3nrux9e8s6dsmf3vg7ngyc4ylgsht9fkmt6j4teq5y2sy9v9lye59ugztr29fw566dsdwu6vk3pw8dmelynkh8rg4fe88y3udntgwdcrzhun0j7kvjgxskvutyjlestwdfue27uqkdp94yhq0pgdxz8zs5tnxdnctzcf6v3yg2zlypesuzf8fatxxtt4564rr6uwdgjnepwv77gmw9erqp4hwtu40x4gxcyvz7pqpk724yd5g44ldh80fj53quhga0re7sqckmr2qfscdunqzu6s3am59t0p7dppw9jq0tfjnq4e8y78ywxhnjyct45g4q4jthn2yqh9gsep2cuuxq6pg3meye7qxs4y598t24su8ky8u25hfcfg9km5j5mf47vq9w8zu7kcaw74rqyq95wxdz84synckuut0ejt0yauzmxcyufjfmvgdx4xphvmf6882av97326hjfvr7qs72umjf6jq5jmhgmjz4vxj3j09ew4xxa3v92dcdqmwa3eshhq939c24l0ymv6lcf33wmcrr4ty869jl2yksjg33sc885q0wvrzn5wkd4ewgjmn660hgfgcs03r40wzugl7u7tssrunrs5du0scu077hc5vee2vxxtqrsgmmvg3h5usg8mcjk4yf8gnx6g6pzy076gv3ysxn6vspp3s6eyk2e39dhpj98r9u656kqa3c2tatkw3uucunru92d4z45yqp8q2wy4py3uyp0n9rsu2dv29xqvhnjh8tw324qc6favu59t7mryvz50d96sq2njfmsdqvqw3fvjxfvvrhldzllfasrlr69qt9ujtdeezycx7axwwc3jt5fhydfq7mcgdt4xuf8v4v6kk7y8fe5xxlyfyuhwlqlvpce0yjr2pnf264m5llgtprkpwa940f4n6ju4kgh0vnstw8lnw99pe9ta06pyccuqkdr6p6dkq3kk7v8mw2pxp0vzex0semjau6cewmtdlluqlkwxx9y6cuf0uaj3262qs640arlw99wfsxgzc3ev3r5fkqvy9v028gtr9mfs7s3yuyrug56qtdx0w76azyt3exfeetuqdxx9dm9k70er2ky9dqyjzfrd0ezesz55h8pqp2pltylyaptt7l9pf92qtj5cye7q2ezk2567v6hwus58rdrxefprcgshscg7ttc4hsact5c57zc6kqretwqs6dxk00qyznfz9uxzuuufekt275mxxcvf8ad4su08u3k5d56gptxcpjsg60d3qm7e4ryyqajxlq6kf4659jhvqv52fvs0ztv9zmrkz3y08hd9snurakf3fth9202gwur8gmrs3g4pdqunmrugk8kttsdxmhyegzwg46vcytuas4q0nsrjxanng6jp896szgtqm6xf5l3v86kyy5lxxtn2ht6j5wy5mkygmwus9fpzkrwqdzht3up28mesr6rfzj3pvepgdyq5nhsqzdrcu2u3xcwnx6ujvght6hgr4qg555dky50uvqxete2xcf4e6udh2d76ulakfe4uv3uxly93p4tpx4fdxgezyq9yxpmk2730akezna9fswmakxzg9446nft0ry2zy2ftnra9lr6m3vx96xszwd44rnrewt0x5fhjj248psmyztt899y28y4h2g5j3veq294np5wqd7vnn27zmkqxzadn5ylj0m3465t4fzzvrmmftdurlcxpxzj0zxmk2nhjw3h7ggtgvgzxe4wasjcfd4xflrzxs7ejp73
+  - &staging_master age1w8tg8mpwj2ujxw0p9k36cpgecq285luwl4wf7a5tjtej4t2wffcq0gnzdw
+
+  # Host Keys (Used by the specific VM to decrypt its own secrets at boot)
+  # - &host_control_center age1...
 
 creation_rules:
-  - path_regex: secrets/[^/]+\.(yaml|json|env|ini)$
+  # Staging Environment
+  # All files under secrets/staging/ are encrypted purely with the staging_master key.
+  # Ephemeral VMs are injected with this private key at boot via Cloud-Init.
+  - path_regex: secrets/staging/.*\.ya?ml$
     key_groups:
-    - age:
-        - *tdpeuter_Tibo-NixFatDesk
-        - *tdpeuter_Tibo-NixTop
+      - age:
+          - *staging_master
+
+  # Production Environment
+  # All files under secrets/prod/ are encrypted with the prod_master key AND the specific host's key.
+  # Ephemeral VMs CANNOT decrypt these files.
+  - path_regex: secrets/prod/.*\.ya?ml$
+    key_groups:
+      - age:
+          - *prod_master
+          # Add host keys here as they are provisioned
+          # - *host_control_center

BOOTSTRAP.md

@@ -0,0 +1,60 @@
+# NixOS GitOps Bootstrap Guide
+
+This repository is designed to be fully automated once bootstrapped, but if you are adapting this codebase for **your own infrastructure**, you must modify several deployment-specific variables before running the bootstrap script on a fresh Proxmox host.
+
+## Adapt the Codebase
+
+Before bootstrapping your host, fork or clone this repository and make the following changes to match your environment:
+
+### Hardware Identifiers
+- **Find your NVMe/Disk UUID**: Log into your fresh Proxmox host and run:
+   ```bash
+   ls -l /dev/disk/by-id/
+   ```
+   Identify your primary data disk (e.g. `nvme-eui...` or `wwn-0x...`).
+- **Update OpenTofu Config**: Open `opentofu/nodes/mikoshi/main.tf` (you may want to rename `mikoshi` to your host's name) and replace the `disk` ID inside the `zpool` resource with your hardware UUID.
+
+### Identity & Access
+- **SSH Keys**: The GitOps Control Center needs an SSH key for disaster recovery. 
+   - Update the Cloud-Init SSH key in `opentofu/nodes/mikoshi/main.tf` under the `user_account` block.
+   - Update the permanent NixOS SSH key in `nixos/users/admin/default.nix`.
+- **Secrets (SOPS)**: Replace the placeholder tokens in the Control Center host config (e.g., `nixos/hosts/izanagi/secrets.yaml` if you haven't renamed it) with your actual Proxmox API token and Forgejo token. Encrypt this file with your own `sops` Age key.
+
+### Hostname & Naming Schema
+If your Proxmox host or your Control Center has a different name:
+- Rename the folders in `opentofu/nodes/` and `nixos/hosts/`.
+- Update the `node_name` inside your OpenTofu `main.tf`.
+- Update `nixos/flake.nix` to reflect your new host names.
+
+### The Golden Image
+Because the GitOps Control Center must be spun up fully configured without human intervention, you need a pre-built NixOS `.qcow2` image.
+1. Build the golden image locally (requires Nix/WSL):
+   ```bash
+   nix run github:nix-community/nixos-generators -- --flake ./nixos#izanagi --format qcow
+   ```
+2. Upload the resulting `.qcow2` file to your Proxmox node.
+3. Create a new VM in Proxmox with **ID 9000**.
+4. Import the `.qcow2` as its disk and convert the VM into a **Template**. *(Ensure the template has Cloud-Init configured).*
+
+## Execute the Bootstrap
+
+Once you have pushed your adapted codebase to your Git server, SSH into your fresh Proxmox node as `root` and run the bootstrap script:
+
+```bash
+curl -fsSL https://git.your-server.com/your-repo/raw/branch/main/scripts/bootstrap.sh | bash
+```
+
+### What this script does automatically:
+- **Fixes APT Repositories**: Disables enterprise repositories and adds community repositories.
+- **Fixes NIC Offloading**: Installs a systemd service to safely disable TSO/GSO/GRO on physical interfaces to prevent network drops.
+- **Installs OpenTofu**: Pulls the official binaries.
+- **Applies Host State**: Runs `tofu apply` which:
+   - Formats your specified disk into the `data` ZFS pool.
+   - Sets laptop lid switch to ignore (if applicable).
+   - Spins up the GitOps Control Center VM.
+
+## Post-Bootstrap
+
+Once the bootstrap script completes, the Control Center VM will boot, initialize via Cloud-Init, and automatically start pulling this git repository.
+
+From this point on, **you no longer need to log into the Proxmox host.** All future changes to VMs, networks, or applications should be done declaratively via Pull Requests to your repository!

DISASTER_RECOVERY.md

@@ -0,0 +1,32 @@
+# Disaster Recovery Plan (Zero-Login Strategy)
+
+In the event of a catastrophic failure, this document outlines how to restore services without causing configuration drift (i.e., without SSHing directly into target nodes and making manual undocumented changes).
+
+## 1. Full Proxmox Host Freeze (Hardware/GPU Bug)
+If the entire Proxmox hypervisor freezes (often caused by the AMD GPU reset bug when a VM reboots):
+*   **DO NOT** attempt to SSH into the host or the VMs.
+*   **Recovery Action**: Use your Out-of-Band Management (OOBM) solution. Access your PiKVM, IPMI interface, or remotely controlled Smart Plug to perform a **hard power cycle** of the host.
+*   Upon reboot, Proxmox will automatically start the VMs, and `comin` will pull the latest working configuration.
+
+## 2. Production Service Failure (Code Bug)
+If a bad PR was merged and a stateless service is failing to start:
+*   **DO NOT** SSH into the VM to fix the config manually.
+*   **Recovery Action**: Open the Forgejo Web UI and click **Revert** on the merged PR.
+*   Within 60 seconds, the `comin` agent running on the target VM will detect the new commit on the `v2` branch, pull the reverted configuration, and restart the service automatically.
+
+## 3. Production Data Corruption (Database/Stateful Bug)
+If a bad deployment corrupted persistent data on TrueNAS (e.g., a failed one-way database schema migration):
+*   **Recovery Action**: 
+    1. Revert the code PR in Forgejo as described in section 2.
+    2. Execute the `scripts/dr-rollback-dataset.sh` script from the secure **Control Center VM**. This script uses the TrueNAS REST API to rollback the specific ZFS dataset to the automated snapshot taken immediately prior to the deployment.
+
+## 4. Total Git Repository Loss
+If the `v2` branch is completely destroyed or the Gitea server is unrecoverable:
+*   Your `prod-master` private age key is stored safely offline on your USB drive.
+*   A recent backup of the repository should be available via your external backup mechanism.
+*   Clone the backup to your local workstation, run the `proxmox-bootstrap` Terraform state locally to ensure the hypervisor is reachable, and use Terraform to redeploy the Control Center VM and Forgejo instances.
+
+## 5. Manually Obtaining the Golden Image
+If the CI pipeline is down and you urgently need to provision a new VM:
+1. Run `nix run github:nix-community/nixos-generators -- --format proxmox -c hosts/Template/default.nix -o result` on your local machine.
+2. The output `.qcow2` image can be uploaded to the Proxmox Web UI manually under `local` -> `ISO Images` (or directly via `qm importdisk` if you must use SSH as a last resort).

NixOS GitOps Migration Specification.md

@@ -0,0 +1,138 @@
+# **Infrastructure Specification: Automated NixOS GitOps Environment**
+
+## **1\. Architecture Overview**
+
+This specification outlines the migration from a manually deployed NixOS environment to a fully automated, pull-based GitOps architecture.
+
+* **Source of Truth:** Forgejo repository (v2 branch).  
+* **Infrastructure Provisioning:** Terraform (via bpg/proxmox provider) executed by Forgejo Actions.  
+* **Configuration Management:** NixOS Flakes, pulled autonomously by individual nodes via comin.  
+* **Secret Management:** sops-nix using Age keys (SSH host keys for VM decryption).  
+* **Storage:** TrueNAS serving persistent data via NFS/iSCSI, utilizing ZFS snapshot cloning for the staging environment.  
+* **Dependency Management:** RenovateBot running via Forgejo Actions.
+
+## **2\. Phase 1: Foundation & State Preparation**
+
+### **2.1 Git Branching Strategy**
+
+* Initialize a new orphan branch named v2 (or a fresh repository) to ensure a clean slate for the flake lockfile and SOPS configuration.  
+* Cherry-pick existing .nix service modules from the legacy main branch as they are migrated.
+
+### **2.2 Secret Management (Bankruptcy & Reset)**
+
+* Delete all legacy .sops.yaml configurations.  
+* Generate a new master age key. This key will be stored securely offline (e.g., Bitwarden) and **never** placed on the Forgejo runner or target VMs.  
+* Initialize a new .sops.yaml. Target VMs will be added via their public SSH host keys as they are provisioned.
+
+### **2.3 Storage Strategy (TrueNAS & Staging)**
+
+To provide the Staging VM with production-accurate data without risking corruption:
+
+1. **Production State:** TrueNAS maintains the primary ZFS datasets.  
+2. **Staging State (On-Demand):** When the staging environment spins up, a script (via Proxmox/Forgejo Action or TrueNAS API) takes a temporary ZFS snapshot of the production dataset.  
+3. **Clone & Mount:** The snapshot is cloned and exported via NFS/SMB specifically for the Staging VM.  
+4. **Teardown:** When Staging is spun down, the ZFS clone is destroyed.
+
+## **3\. Phase 2: Proxmox Configuration & Least Privilege**
+
+Terraform requires API access to Proxmox. To adhere to the principle of least privilege, Proxmox RBAC (Role-Based Access Control) will be utilized.
+
+### **3.1 Proxmox Static Configuration (RBAC)**
+
+The root user (root@pam) must **never** be used for automation. Instead, a dedicated API user with a highly restricted role must be created. This can be done via the Proxmox Web UI (Datacenter \-\> Permissions) or via the Proxmox shell (pveum).  
+**1\. Create the Restricted Role (TerraformProv):**  
+This role grants only the permissions needed to clone templates, configure hardware, and manage power states.  
+pveum role add TerraformProv \-privs "VM.Allocate VM.Audit VM.Clone VM.Config.CDROM VM.Config.CPU VM.Config.Disk VM.Config.HWType VM.Config.Memory VM.Config.Network VM.Config.Options VM.Monitor VM.PowerMgmt Datastore.AllocateSpace Datastore.Audit SDN.Use"
+
+**2\. Create the Terraform User:**  
+Create a user in the Proxmox VE authentication realm (@pve).  
+pveum user add terraform@pve
+
+**3\. Assign Permissions to Paths:**  
+Bind the role to the user, strictly limiting access to the VMs path and the specific storage path where VMs will reside (e.g., local-zfs).  
+pveum acl modify /vms \-user terraform@pve \-role TerraformProv  
+pveum acl modify /storage/local-zfs \-user terraform@pve \-role TerraformProv
+
+**4\. Generate API Token:**  
+Generate a token for the user. Proxmox will output a Secret ID. This ID is saved into Forgejo Secrets and passed to the Terraform provider.  
+pveum user token add terraform@pve tf-automation \--privsep 0
+
+*(Note: \--privsep 0 ensures the token inherits the user's permissions, rather than requiring separate ACL definitions).*
+
+## **4\. Phase 3: Infrastructure as Code (Terraform)**
+
+### **4.1 The Golden Image (.qcow2)**
+
+A minimal NixOS image is built locally using nixos-generators and uploaded to Proxmox as a template (e.g., ID 9000). This template contains:
+
+* QEMU Guest Agent.  
+* Cloud-Init support.  
+* comin installed.
+
+### **4.2 Terraform Provisioning**
+
+Terraform maintains the hardware state. Its responsibilities are strictly limited to:
+
+* Cloning Template 9000\.  
+* Allocating vCPU, RAM, and Virtual Disks.  
+* Using Cloud-Init to inject network configurations (IP, Gateway) and the Forgejo runner's public SSH key for health check access.
+
+## **5\. Phase 4: CI/CD & Automation**
+
+### **5.1 Forgejo Runner & Terraform Pipeline**
+
+* The newly provisioned Forgejo Runner VM registers with Forgejo.  
+* A workflow (terraform.yml) runs terraform plan on Pull Requests and terraform apply on merges to the main branch.
+
+### **5.2 GitOps Pull Agent (comin and Alternatives)**
+
+Once provisioned by Terraform, VMs boot and pull their OS configuration autonomously.
+
+* **Chosen Tool (comin):** Polling agent designed specifically for NixOS GitOps. Runs as a systemd service, supports multiple git remotes, branch tracking, and local cryptographic verification.  
+* **Alternative 1 (system.autoUpgrade):** The native NixOS approach utilizing systemd timers. *Rejected* because it lacks advanced Git authentication and webhooks for immediate triggering.  
+* **Alternative 2 (deploy-rs / colmena):** Excellent push-based deployment tools. *Rejected* because they require the CI runner to maintain SSH root access to the entire fleet, violating our zero-trust/pull-only goals.
+
+### **5.3 RenovateBot Integration**
+
+* Renovate is deployed via a Forgejo Action running on a cron schedule.  
+* Nix flake support is explicitly enabled in renovate.json.  
+* Renovate parses flake.lock and Docker tags, automatically opening PRs for updates.
+
+## **6\. Phase 5: Staging Lifecycle & Healthchecks**
+
+### **6.1 Lifecycle Automation**
+
+1. **Trigger:** A PR is opened by Renovate or a developer.  
+2. **Wake-up:** A Forgejo Action calls the Proxmox API to start the nixos-staging VM and triggers the TrueNAS ZFS snapshot clone.  
+3. **Pull:** The Staging VM boots, connects to the network, and comin pulls the PR's commit hash.
+
+### **6.2 Automated Testing**
+
+The Forgejo Action will verify the deployment was successful:
+
+* **Systemd Checks:** systemctl is-system-running \--wait and systemctl is-active \<service\>.  
+* **Docker Healthchecks:** docker inspect \--format='{{json .State.Health.Status}}' \<container\>.  
+* **HTTP Probes:** Execute curl commands against the Staging VM to verify HTTP 200 OK.
+
+### **6.3 Spin Down**
+
+Upon PR merge or closure, the Forgejo Action shuts down the VM via the Proxmox API and destroys the temporary TrueNAS ZFS clone.
+
+## **7\. Phase 6: Rollbacks & Disaster Recovery**
+
+Because infrastructure involves both *stateless* configurations (NixOS) and *stateful* data (TrueNAS), a unified rollback strategy is critical.
+
+### **7.1 Differentiating Failures (Nix vs. TrueNAS)**
+
+If a deployment fails, the root cause must be identified to determine the rollback path:
+
+* **Stateless Failure (NixOS/Code):** The service fails to start immediately. System logs (journalctl \-u my-service) show syntax errors, missing binaries, or bad systemd unit definitions.  
+* **Stateful Failure (TrueNAS/Data):** The service starts but crashes with a database schema version mismatch, "permission denied" on the NFS mount, or missing user data. This means a service (like a Docker container) attempted a one-way database migration on the persistent TrueNAS dataset and failed halfway.
+
+### **7.2 The Unified Rollback Procedure**
+
+When a production rollout fails, **do not** attempt to fix it live.
+
+1. **Revert the State (TrueNAS):** If the failure was stateful (e.g., a bad database migration), immediately log into TrueNAS and rollback the primary dataset to the automated ZFS snapshot taken right before the deployment.  
+2. **Revert the Code (Forgejo):** Use the Forgejo UI to click "Revert" on the problematic Pull Request. This creates a new commit restoring the previous flake.nix state.  
+3. **Autonomous Recovery:** Within 60 seconds, the production VM's comin agent will detect the new commit on main, pull the reverted code, apply the old configuration, and reconnect to the restored TrueNAS dataset.

README.md

@@ -0,0 +1,28 @@
+# Bos55 Homelab (NixOS + Proxmox GitOps)
+
+This repository contains the declarative infrastructure-as-code for the Bos55 Homelab. It leverages NixOS, Flakes, Terraform, and a custom native pull-based GitOps architecture to fully automate the provisioning, configuration, and maintenance of a home server environment.
+
+## Goals & Philosophy
+
+The primary goal of this project is **Zero-Login**. 
+Servers should be treated as cattle, not pets. If a configuration needs to change, it must be changed in this repository. 
+
+* **Declarative Infrastructure:** Every application, user, reverse proxy rule, and network configuration is defined declaratively using custom NixOS modules.
+* **Autonomous Pull-Based Deployments:** Nodes autonomously pull updates from this repository via a custom `homelab.gitops` systemd service, securely offloading compilation to a dedicated Build farm.
+* **Instant Webhooks:** Merging a Pull Request instantly triggers deployments across the cluster via cryptographic webhooks.
+* **Micro-segmented Security:** Ephemeral staging environments are completely isolated from production datasets using Proxmox hypervisor-level firewalls.
+* **Secret Management:** Strict separation between production and staging secrets using `sops-nix` and `age`.
+
+## Quickstart
+
+If you are setting up this repository from scratch, you **must** perform the initial bootstrap before the automated pipelines can function.
+
+1. **Bootstrap Keys:** Follow the [BOOTSTRAP.md](./BOOTSTRAP.md) guide to generate your offline `age` master keys and configure Proxmox API access.
+2. **Deploy Builder:** Ensure the dedicated Nix `Builder` host is running so other nodes can securely offload package compilation.
+3. **Commit & Push:** Make your configuration changes to `flake.nix` or the `hosts/` directory, and push to the `v2` branch.
+4. **Autonomous Deployment:** The nodes will automatically fetch the new hash and apply the configuration.
+
+## Documentation
+
+* [Bootstrap Guide](./BOOTSTRAP.md) - Initial setup instructions.
+* [Disaster Recovery](./DISASTER_RECOVERY.md) - Protocols for handling hardware freezes and ZFS corruption.

flake.nix

@@ -1,23 +1,30 @@
 {
-  description = "Homelab configuration using flakes";
+  description = "Homelab configuration using flakes (v2 GitOps)";
 
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
 
     flake-utils.url = "github:numtide/flake-utils";
+    
     sops-nix = {
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    
     utils = {
       url = "github:gytis-ivaskevicius/flake-utils-plus";
       inputs.flake-utils.follows = "flake-utils";
     };
+
+    pre-commit-hooks = {
+      url = "github:cachix/pre-commit-hooks.nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs = inputs@{
     self, nixpkgs,
-    flake-utils, sops-nix, utils,
+    flake-utils, sops-nix, utils, pre-commit-hooks,
     ...
   }:
   let
@@ -26,39 +33,45 @@
   utils.lib.mkFlake {
     inherit self inputs;
 
+    outputsBuilder = channels: let
+      pkgs = channels.nixpkgs;
+      system = pkgs.system;
+      pre-commit-check = pre-commit-hooks.lib.${system}.run {
+        src = ./.;
+        hooks = {
+          nixfmt-rfc-style.enable = true;
+          statix.enable = true;
+          terraform-format.enable = true;
+        };
+      };
+    in {
+      checks = {
+        inherit pre-commit-check;
+      };
+      devShells.default = pkgs.mkShell {
+        inherit (pre-commit-check) shellHook;
+        buildInputs = pre-commit-check.enabledPackages;
+      };
+    };
+
     hostDefaults = {
       inherit system;
 
       modules = [
-        ./modules
-        ./users
+        ./nixos/modules
+        ./nixos/users
 
         sops-nix.nixosModules.sops
+        ({ config, pkgs, ... }: {
+          # Inject git revision for our custom GitOps module to query
+          system.configurationRevision = self.rev or self.dirtyRev or "unknown";
+        })
       ];
     };
 
     hosts = {
-      # Physical hosts
-      Niko.modules          = [ ./hosts/Niko          ];
-
-      # Virtual machines
-
-      # Single-service
-      Ingress.modules       = [ ./hosts/Ingress       ];
-      Gitea.modules         = [ ./hosts/Gitea         ];
-      Vaultwarden.modules   = [ ./hosts/Vaultwarden   ];
-
-      # Production multi-service
-      Binnenpost.modules    = [ ./hosts/Binnenpost    ];
-      Production.modules    = [ ./hosts/Production    ];
-      ProductionGPU.modules = [ ./hosts/ProductionGPU ];
-      ProductionArr.modules = [ ./hosts/ProductionArr ];
-      ACE.modules           = [ ./hosts/ACE           ];
-
-      # Others
-      Template.modules      = [ ./hosts/Template      ];
-      Development.modules   = [ ./hosts/Development   ];
-      Testing.modules       = [ ./hosts/Testing       ];
+      izanagi.modules = [ ./nixos/hosts/izanagi ];
+      Gitea.modules = [ ./nixos/hosts/Gitea ];
     };
   };
 }

nixos/hosts/Builder/default.nix

@@ -0,0 +1,50 @@
+{ pkgs, ... }:
+
+{
+  config = {
+    networking = {
+      hostName = "Builder";
+      hostId = "aaaa4200";
+      domain = "depeuter.dev";
+
+      useDHCP = false;
+      enableIPv6 = true;
+
+      defaultGateway = {
+        address = "192.168.0.1";
+        interface = "ens18";
+      };
+
+      interfaces.ens18 = {
+        ipv4.addresses = [
+          {
+            address = "192.168.0.42";
+            prefixLength = 24;
+          }
+        ];
+      };
+
+      nameservers = [
+        "1.1.1.1"
+        "1.0.0.1"
+      ];
+    };
+
+    # Dedicated builder user
+    users.users.builder = {
+      isNormalUser = true;
+      description = "Nix Remote Builder";
+      # You must add the public key corresponding to `builder-ssh-key` here
+      openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1... TODO: replace with build-farm-key.pub"
+      ];
+    };
+
+    # Optimize nix settings for a build farm
+    nix.settings.trusted-users = [ "builder" ];
+    nix.settings.cores = 0; # Use all cores
+    nix.settings.max-jobs = "auto";
+
+    system.stateVersion = "24.05";
+  };
+}

nixos/hosts/izanagi/default.nix

@@ -0,0 +1,52 @@
+{ config, pkgs, ... }:
+
+{
+  networking.hostName = "izanagi";
+
+  homelab = {
+    # Enable the standard guest VM configuration
+    virtualisation.guest.enable = true;
+
+    # Enable standard GitOps for the Control Center itself
+    gitops.enable = true;
+    # We might pull from our Forgejo instance eventually, but for bootstrap
+    # it might need to pull from Github or the local Gitea if it's up.
+    # gitops.repoUrl = "https://git.depeuter.dev/Bos55/nix-config.git";
+
+    services = {
+      openssh.enable = true;
+
+      # Enable the Hypervisor GitOps service to provision OTHER VMs
+      hypervisor-gitops = {
+        enable = true;
+        repoUrl = "https://git.depeuter.dev/Bos55/nix-config.git";
+        pollInterval = "hourly";
+      };
+    };
+  };
+
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    defaultSopsFormat = "yaml";
+    age.keyFile = "/var/lib/sops-age/keys.txt"; # Injected via Cloud-Init during Phase 4
+
+    secrets = {
+      proxmox_api_token.owner = "root";
+      forgejo_token.owner = "root";
+    };
+
+    templates."hypervisor-gitops.env".content = ''
+      PROXMOX_VE_API_TOKEN=${config.sops.placeholder.proxmox_api_token}
+      FORGEJO_TOKEN=${config.sops.placeholder.forgejo_token}
+      PROXMOX_VE_ENDPOINT=https://mikoshi:8006/
+      PROXMOX_VE_INSECURE=true
+    '';
+  };
+
+  # Make the secrets available to the hypervisor-gitops service via EnvironmentFile
+  systemd.services.hypervisor-gitops.serviceConfig.EnvironmentFile = [
+    config.sops.templates."hypervisor-gitops.env".path
+  ];
+
+  system.stateVersion = "24.05";
+}

nixos/hosts/izanagi/secrets.yaml

@@ -0,0 +1,7 @@
+# This is a placeholder SOPS file.
+# You must encrypt it with sops using your age key before deploying.
+# sops -e -i secrets.yaml
+proxmox_api_token: "PLACEHOLDER_TOKEN"
+forgejo_token: "PLACEHOLDER_TOKEN"
+builder-ssh-key: "PLACEHOLDER_KEY"
+webhook-secret: "PLACEHOLDER_SECRET"

nixos/modules/apps/arr/default.nix

@@ -61,34 +61,30 @@ in {
       virtualisation.containers.enable = lib.mkIf inUse true;
     };
 
-    fileSystems = let
-      mkFileSystem = device: {
-        inherit device;
-        fsType = "nfs";
-        options = [
-          "rw"
-          "auto"
-          "nfsvers=4.2"
-          "rsize=1048576" "wsize=1048576"
-          "hard"
-          "timeo=600" "retrans=2"
-          "_netdev" "nosuid" "tcp"
-        ];
-      };
-
+    homelab.nfsMounts = let
       hugoBackup = "192.168.0.11:/mnt/BIG/BACKUP";
+      arrOptions = [
+        "auto"
+        "rsize=1048576" "wsize=1048576"
+        "hard"
+        "timeo=600" "retrans=2"
+        "_netdev"
+      ];
+      mkMount = device: {
+        inherit device;
+        extraOptions = arrOptions;
+      };
     in lib.mkIf inUse {
-      "/srv/bazarr-backup" = lib.mkIf cfg.bazarr.enable (mkFileSystem "${hugoBackup}/BAZARR");
-      "/srv/prowlarr-backup" = lib.mkIf cfg.bazarr.enable (mkFileSystem "${hugoBackup}/PROWLARR");
-      "/srv/qbittorrent" = lib.mkIf cfg.qbittorrent.enable (mkFileSystem "192.168.0.11:/mnt/SMALL/CONFIG/QBITTORRENT");
-      "/srv/radarr-backup" = lib.mkIf cfg.radarr.enable (mkFileSystem "${hugoBackup}/RADARR");
-      "/srv/sonarr-backup" = lib.mkIf cfg.sonarr.enable (mkFileSystem "${hugoBackup}/SONARR");
-      "/srv/torrent" = mkFileSystem "192.168.0.11:/mnt/SMALL/MEDIA/TORRENT";
+      "/srv/bazarr-backup" = lib.mkIf cfg.bazarr.enable (mkMount "${hugoBackup}/BAZARR");
+      "/srv/prowlarr-backup" = lib.mkIf cfg.bazarr.enable (mkMount "${hugoBackup}/PROWLARR");
+      "/srv/qbittorrent" = lib.mkIf cfg.qbittorrent.enable (mkMount "192.168.0.11:/mnt/SMALL/CONFIG/QBITTORRENT");
+      "/srv/radarr-backup" = lib.mkIf cfg.radarr.enable (mkMount "${hugoBackup}/RADARR");
+      "/srv/sonarr-backup" = lib.mkIf cfg.sonarr.enable (mkMount "${hugoBackup}/SONARR");
+      "/srv/torrent" = mkMount "192.168.0.11:/mnt/SMALL/MEDIA/TORRENT";
     };
 
     # Make sure the Docker network exists.
-    systemd.services."docker-${networkName}-create-network" = lib.mkIf inUse {
-      description = "Create Docker network for ${networkName}";
+    homelab.dockerNetworks."${networkName}" = lib.mkIf inUse {
       requiredBy = [
         "docker-bazarr.service"
         "docker-prowlarr.service"
@@ -96,38 +92,47 @@ in {
         "docker-radarr.service"
         "docker-sonarr.service"
       ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-      script = ''
-        if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
-          ${pkgs.docker}/bin/docker network create ${networkName}
-        fi
-      '';
     };
 
     # Create a user for each app.
-    users.users = let
+    homelab.appUsers = let
       mkUser = uid: {
-        uid = lib.mkForce uid;
-        isSystemUser = true;
+        inherit uid;
         group = config.users.groups.media.name;
-        home = "/var/empty";
-        shell = null;
       };
     in {
       bazarr = lib.mkIf cfg.bazarr.enable (mkUser 3003);
       prowlarr = lib.mkIf cfg.prowlarr.enable (mkUser 3004);
-      qbittorrent = lib.mkIf cfg.qbittorrent.enable (mkUser 3005) // {
-        extraGroups = [
-          config.users.groups.apps.name
-        ];
-      };
+      qbittorrent = lib.mkIf cfg.qbittorrent.enable ((mkUser 3005) // {
+        extraGroups = [ config.users.groups.apps.name ];
+      });
       radarr = lib.mkIf cfg.radarr.enable (mkUser 3006);
       sonarr = lib.mkIf cfg.sonarr.enable (mkUser 3007);
     };
 
+    homelab.traefikRouters = {
+      bazarr = lib.mkIf cfg.bazarr.enable {
+        rule = "Host(`bazarr.depeuter.dev`)";
+        port = 6767;
+      };
+      prowlarr = lib.mkIf cfg.prowlarr.enable {
+        rule = "Host(`prowlarr.depeuter.dev`)";
+        port = 9696;
+      };
+      qbittorrent = lib.mkIf cfg.qbittorrent.enable {
+        rule = "Host(`qb.depeuter.dev`)";
+        port = 10095;
+      };
+      radarr = lib.mkIf cfg.radarr.enable {
+        rule = "Host(`radarr.depeuter.dev`)";
+        port = 7878;
+      };
+      sonarr = lib.mkIf cfg.sonarr.enable {
+        rule = "Host(`sonarr.depeuter.dev`)";
+        port = 8989;
+      };
+    };
+
     virtualisation.oci-containers.containers = let
       videoHostPath = config.homelab.fileSystems.media.video.hostPath;
     in {
@@ -159,12 +164,6 @@ in {
           "${videoHostPath}/Films:/media/movies"
           "${videoHostPath}/Series:/media/series"
         ];
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.docker.network" = proxyNet;
-          "traefik.http.routers.bazarr.rule" = "Host(`bazarr.depeuter.dev`)";
-          "traefik.http.services.bazarr.loadbalancer.server.port" = toString port;
-        };
       };
 
       prowlarr = let
@@ -190,12 +189,6 @@ in {
 
           "/srv/prowlarr-backup:/config/Backups"
         ];
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.docker.network" = proxyNet;
-          "traefik.http.routers.prowlarr.rule" = "Host(`prowlarr.depeuter.dev`)";
-          "traefik.http.services.prowlarr.loadbalancer.server.port" = toString port;
-        };
       };
 
       qbittorrent = let
@@ -223,12 +216,6 @@ in {
 
           "/srv/torrent:/media/cache"
         ];
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.docker.network" = proxyNet;
-          "traefik.http.routers.qbittorrent.rule" = "Host(`qb.depeuter.dev`)";
-          "traefik.http.services.qbittorrent.loadbalancer.server.port" = toString port;
-        };
       };
 
       radarr = let
@@ -257,12 +244,6 @@ in {
           "/srv/torrent:/media/cache"
           "${videoHostPath}/Films:/media/movies"
         ];
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.docker.network" = proxyNet;
-          "traefik.http.routers.radarr.rule" = "Host(`radarr.depeuter.dev`)";
-          "traefik.http.services.radarr.loadbalancer.server.port" = toString port;
-        };
       };
 
       sonarr = let
@@ -291,12 +272,6 @@ in {
           "/srv/torrent:/media/cache"
           "${videoHostPath}/Series:/media/series"
         ];
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.docker.network" = proxyNet;
-          "traefik.http.routers.sonarr.rule" = "Host(`sonarr.depeuter.dev`)";
-          "traefik.http.services.sonarr.loadbalancer.server.port" = toString port;
-        };
       };
     };
   };

nixos/modules/apps/calibre/default.nix

@@ -34,58 +34,48 @@ in {
         virtualisation.containers.enable = true;
       };
 
-      users.users.calibre = {
-        uid = lib.mkForce 3010;
-        isSystemUser = true;
+      homelab.appUsers.calibre = {
+        uid = 3010;
         group = config.users.groups.media.name;
-        home = "/var/empty";
-        shell = null;
       };
 
-      fileSystems."${books}" = {
+      homelab.nfsMounts."${books}" = {
         device = "192.168.0.11:/mnt/SMALL/MEDIA/BOOKS";
-        fsType = "nfs";
-        options = [
-          "rw"
+        extraOptions = [
           "auto"
-          "nfsvers=4.2"
           "rsize=1048576" "wsize=1048576"
-          "soft"
           "timeo=600" "retrans=2"
-          "_netdev" "nosuid" "tcp"
+          "_netdev"
         ];
       };
 
       # Make sure the Docker network exists.
-      systemd.services."docker-${networkName}-create-network" = {
+      homelab.dockerNetworks."${networkName}" = {
         requiredBy = [
           "docker-calibre.service"
         ];
-        serviceConfig = {
-          Type = "oneshot";
-          RemainAfterExit = true;
+      };
+      homelab.traefikRouters = {
+        calibre = lib.mkIf cfg.desktop.enable {
+          rule = "Host(`calibre.depeuter.dev`)";
+          port = 8080;
+        };
+        calibre-web = lib.mkIf cfg.web.enable {
+          rule = "Host(`books.depeuter.dev`)";
+          port = 8083;
         };
-        script = ''
-          if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
-            ${pkgs.docker}/bin/docker network create ${networkName}
-          fi
-        '';
       };
     })
 
     # Calibre desktop
     (lib.mkIf cfg.desktop.enable {
-      fileSystems."${calibre-config}" = {
+      homelab.nfsMounts."${calibre-config}" = {
         device = "192.168.0.11:/mnt/SMALL/CONFIG/CALIBRE";
-        fsType = "nfs";
-        options = [
-          "rw"
+        extraOptions = [
           "auto"
-          "nfsvers=4.2"
           "rsize=1048576" "wsize=1048576"
-          "soft"
           "timeo=600" "retrans=2"
-          "_netdev" "nosuid" "tcp"
+          "_netdev"
         ];
       };
 
@@ -122,28 +112,18 @@ in {
 
           "${books}:/media/books"
         ];
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.docker.network" = proxyNet;
-          "traefik.http.routers.calibre.rule" = "Host(`calibre.depeuter.dev`)";
-          "traefik.http.services.calibre.loadbalancer.server.port" = toString innerPort;
-        };
       };
     })
 
     # Calibre Web
     (lib.mkIf cfg.web.enable {
-      fileSystems."${calibre-web-config}" = {
+      homelab.nfsMounts."${calibre-web-config}" = {
         device = "192.168.0.11:/mnt/SMALL/CONFIG/CALIBRE-WEB";
-        fsType = "nfs";
-        options = [
-          "rw"
+        extraOptions = [
           "auto"
-          "nfsvers=4.2"
           "rsize=1048576" "wsize=1048576"
-          "soft"
           "timeo=600" "retrans=2"
-          "_netdev" "nosuid" "tcp"
+          "_netdev"
         ];
       };
 
@@ -177,12 +157,6 @@ in {
 
           "${books}:/media/books"
         ];
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.docker.network" = proxyNet;
-          "traefik.http.routers.calibre-web.rule" = "Host(`books.depeuter.dev`)";
-          "traefik.http.services.calibre-web.loadbalancer.server.port" = toString innerPort;
-        };
       };
     })
   ];

nixos/modules/apps/default.nix

@@ -8,6 +8,7 @@
     ./gitea
     ./homepage
     ./jellyfin
+    ./monitoring
     ./plex
     ./solidtime
     ./speedtest

nixos/modules/apps/freshrss/default.nix

@@ -4,6 +4,7 @@ let
   cfg = config.homelab.apps.freshrss;
 
   networkName = "freshrss";
+  proxyNet = config.homelab.apps.traefik.sharedNetworkName;
 in {
   options.homelab.apps.freshrss = {
     enable = lib.mkEnableOption "FreshRSS";
@@ -20,35 +21,36 @@ in {
     lib.mkIf cfg.enable {
     homelab.virtualisation.containers.enable = true;
 
-    fileSystems."/srv/freshrss" = {
+    homelab.nfsMounts."/srv/freshrss" = {
       device = "192.168.0.11:/mnt/SMALL/CONFIG/FRESHRSS";
-      fsType = "nfs";
-      options = [
-        "rw"
+      extraOptions = [
         "auto"
-        "nfsvers=4.2"
-        "async" "soft" "timeo=600"
+        "timeo=600"
         "retrans=2"
         "_netdev"
-        "nosuid"
-        "tcp"
       ];
     };
 
-    systemd.services."docker-${networkName}-create-network" = {
-      description = "Create Docker network for ${networkName}";
+    homelab.dockerNetworks."${networkName}" = {
       requiredBy = [
         "docker-freshrss.service"
       ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
+    };
+
+    homelab.traefikRouters.freshrss = {
+      rule = "Host(`rss.depeuter.dev`)";
+      port = 80;
+      tls = true;
+      entryPoints = [ "websecure" ];
+      middlewares = [ "freshrssM1" "freshrssM2" ];
+      extraLabels = {
+        "traefik.http.middlewares.freshrssM1.compress" = "true";
+        "traefik.http.middlewares.freshrssM2.headers.browserXssFilter" = "true";
+        "traefik.http.middlewares.freshrssM2.headers.forceSTSHeader" = "true";
+        "traefik.http.middlewares.freshrssM2.headers.frameDeny" = "true";
+        "traefik.http.middlewares.freshrssM2.headers.referrerPolicy" = "no-referrer-when-downgrade";
+        "traefik.http.middlewares.freshrssM2.headers.stsSeconds" = "31536000";
       };
-      script = ''
-        if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
-          ${pkgs.docker}/bin/docker network create ${networkName}
-        fi
-      '';
     };
 
     virtualisation.oci-containers.containers.freshrss = {
@@ -61,6 +63,7 @@ in {
       ];
       extraOptions = [
         "--network=${networkName}"
+        "--network=${proxyNet}"
       ];
       environment = {
         TZ = config.time.timeZone;
@@ -72,22 +75,6 @@ in {
         "/srv/freshrss/www/freshrss/data:/var/www/FreshRSS/data"
         "/srv/freshrss/www/freshrss/extensions:/var/www/FreshRSS/extensions"
       ];
-      labels = {
-        "traefik.enable" = "true";
-      
-        "traefik.http.middlewares.freshrssM1.compress" = "true";
-        "traefik.http.middlewares.freshrssM2.headers.browserXssFilter" = "true";
-        "traefik.http.middlewares.freshrssM2.headers.forceSTSHeader" = "true";
-        "traefik.http.middlewares.freshrssM2.headers.frameDeny" = "true";
-        "traefik.http.middlewares.freshrssM2.headers.referrerPolicy" = "no-referrer-when-downgrade";
-        "traefik.http.middlewares.freshrssM2.headers.stsSeconds" = "31536000";
-        "traefik.http.routers.freshrss.entryPoints" = "websecure";
-        "traefik.http.routers.freshrss.tls" = "true";
-
-        "traefik.http.services.freshrss.loadbalancer.server.port" = "80";
-        "traefik.http.routers.freshrss.middlewares" = "freshrssM1,freshrssM2";
-        "traefik.http.routers.freshrss.rule" = "Host(`rss.depeuter.dev`)";
-      };
     };
   };
 }

nixos/modules/apps/gitea/default.nix

@@ -21,6 +21,51 @@ in {
   options.homelab.apps.gitea.enable = lib.mkEnableOption "Gitea";
 
   config = lib.mkIf cfg.enable {
+    services.nats = {
+      enable = true;
+      port = 4222;
+      jetstream = {
+        enable = true;
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 4222 9000 ];
+
+    systemd.services.gitea-webhook-bridge = {
+      description = "Gitea Webhook Bridge to NATS JetStream";
+      after = [ "network.target" "nats.service" ];
+      wants = [ "nats.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path = with pkgs; [ natscli python3 ];
+      script = ''
+        python3 -c '
+import http.server
+import subprocess
+import sys
+
+class WebhookHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = int(self.headers.get("Content-Length", 0))
+        body = self.rfile.read(content_length)
+        try:
+            subprocess.run(["nats", "pub", "--server=nats://127.0.0.1:4222", "forgejo.staging"], input=body, check=True)
+            self.send_response(200)
+            self.end_headers()
+            self.wfile.write(b"OK\n")
+        except Exception as e:
+            self.send_response(500)
+            self.end_headers()
+            self.wfile.write(str(e).encode("utf-8"))
+
+    def log_message(self, format, *args):
+        sys.stderr.write("%s - - [%s] %s\n" % (self.client_address[0], self.log_date_time_string(), format%args))
+
+server = http.server.ThreadingHTTPServer(("0.0.0.0", 9000), WebhookHandler)
+server.serve_forever()
+'
+      '';
+    };
+
     homelab = {
       users = {
         apps.enable = true;
@@ -30,61 +75,28 @@ in {
       virtualisation.containers.enable = true;
     };
 
-    users.users.gitea = {
-      uid = lib.mkForce UID;
-      isSystemUser = true;
-      group = config.users.groups.apps.name;
-      home = "/var/empty";
-      shell = null;
+    homelab.appUsers.gitea = {
+      uid = UID;
     };
 
     # Use filesystem mounts because rootless containers otherwise don't have access to the mount path (nested in docker directories).
     # You could probably fix this by modifying the access rights on the path, but what would the point of that be?
-    fileSystems = {
-      # Mount options:
-      # - hard: retry requests indefinitely if the server becomes unresponsive.
-      # - nosuid: prevent set-user-id and set-group-id bits
+    homelab.nfsMounts = {
       "/srv/gitea-config" = {
         device = "192.168.0.11:/mnt/SMALL/CONFIG/GITEA";
-        fsType = "nfs";
-        options = [
-          "rw"
-          "nfsvers=4.2"
-          "async" "soft" "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all"
-          "nosuid"
-          "tcp"
-        ];
       };
 
       "/srv/gitea-git" = {
         device = "192.168.0.11:/mnt/SMALL/DATA/GIT";
-        fsType = "nfs";
-        options = [
-          "rw"
-          "nfsvers=4.2"
-          "async" "soft" "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all"
-          "nosuid"
-          "tcp"
-        ];
       };
     };
 
     # Make sure the Docker network exists.
-    systemd.services."docker-${networkName}-create-network" = {
-      description = "Create Docker network for ${networkName}";
+    homelab.dockerNetworks."${networkName}" = {
       requiredBy = [
         "docker-gitea-db.service"
         "docker-gitea.service"
       ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-      script = ''
-        if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
-          ${pkgs.docker}/bin/docker network create ${networkName}
-        fi
-      '';
     };
 
     virtualisation.oci-containers.containers = {
@@ -471,6 +483,7 @@ in {
           # ... oath2_client
 
           # ... webhook
+          FORGEJO__webhook__ALLOWED_HOST_LIST = "192.168.0.0/16,127.0.0.0/8,host.docker.internal,*";
 
           FORGEJO__mailer__ENABLED = "true";
           # Buffer length of channel, keep it as it is if you don't know what it is.

nixos/modules/apps/homepage/default.nix

@@ -26,26 +26,24 @@ in {
       virtualisation.containers.enable = true;
     };
 
-    users.users.homepage = {
-      uid = lib.mkForce 3018;
-      isSystemUser = true;
-      group = config.users.groups.apps.name;
-      home = "/var/empty";
-      shell = null;
+    homelab.appUsers.homepage = {
+      uid = 3018;
     };
 
-    fileSystems."${homepage-config}" = {
+    homelab.nfsMounts."${homepage-config}" = {
       device = "192.168.0.11:/mnt/SMALL/CONFIG/HOMEPAGE";
-      fsType = "nfs";
-      options = [
-        "rw"
+      extraOptions = [
         "auto"
-        "nfsvers=4.2"
-        "async" "soft" "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all"
-        "nosuid" "tcp"
       ];
     };
 
+    homelab.traefikRouters.homepage = let
+      host = "homepage.${config.networking.domain}";
+    in {
+      rule = "Host(`${host}`)";
+      port = cfg.port;
+    };
+
     virtualisation.oci-containers.containers.homepage = let
       host = "homepage.${config.networking.domain}";
     in {
@@ -63,12 +61,6 @@ in {
         "${homepage-config}:/app/config"
         # "/var/run/docker.sock:/var/run/docker.sock:ro" # For docker integrations
       ];
-      labels = {
-        "traefik.enable" = "true";
-        "traefik.docker.network" = proxyNet;
-        "traefik.http.routers.homepage.rule" = "Host(`${host}`)";
-        "traefik.http.services.homepage.loadbalancer.server.port" = toString cfg.port;
-      };
       environment = {
         inherit PUID PGID;
 

nixos/modules/apps/jellyfin/default.nix

@@ -4,6 +4,7 @@ let
   cfg = config.homelab.apps.jellyfin;
 
   networkName = "jellyfin";
+  proxyNet = config.homelab.apps.traefik.sharedNetworkName;
   inherit (config.homelab.fileSystems) media;
 
   UID = 3008;
@@ -25,71 +26,36 @@ in {
       virtualisation.containers.enable = true;
     };
 
-    fileSystems = {
-      "/srv/audio" = {
-        device = "192.168.0.11:/mnt/SMALL/MEDIA/AUDIO";
-        fsType = "nfs";
-        options = [
-          "ro"
-          "nfsvers=4.2"
-          "async" "soft"
-          "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all"
-          "nosuid" "tcp"
-        ];
-      };
-
-      "/srv/homevideo" = {
-        device = "192.168.0.11:/mnt/BIG/MEDIA/HOMEVIDEO/ARCHIVE";
-        fsType = "nfs";
-        options = [
-          "ro"
-          "nfsvers=4.2"
-          "async" "soft"
-          "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all"
-          "nosuid" "tcp"
-        ];
-      };
-
-      "/srv/photo" = {
-        device = "192.168.0.11:/mnt/BIG/MEDIA/PHOTO/ARCHIVE";
-        fsType = "nfs";
-        options = [
-          "ro"
-          "nfsvers=4.2"
-          "async" "soft"
-          "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all"
-          "nosuid" "tcp"
-        ];
+    homelab.nfsMounts = let
+      mkMount = device: {
+        inherit device;
+        readOnly = true;
       };
+    in {
+      "/srv/audio" = mkMount "192.168.0.11:/mnt/SMALL/MEDIA/AUDIO";
+      "/srv/homevideo" = mkMount "192.168.0.11:/mnt/BIG/MEDIA/HOMEVIDEO/ARCHIVE";
+      "/srv/photo" = mkMount "192.168.0.11:/mnt/BIG/MEDIA/PHOTO/ARCHIVE";
     };
 
-    users.users.jellyfin = {
-      uid = lib.mkForce UID;
-      isSystemUser = true;
-      group = config.users.groups.apps.name;
-      extraGroups = [
-        config.users.groups.media.name
-      ];
-      home = "/var/empty";
-      shell = null;
+    homelab.appUsers.jellyfin = {
+      uid = UID;
+      extraGroups = [ config.users.groups.media.name ];
     };
 
     # Make sure the Docker network exists.
-    systemd.services."docker-${networkName}-create-network" = {
-      description = "Create Docker network for ${networkName}";
+    homelab.dockerNetworks."${networkName}" = {
       requiredBy = [
         "docker-jellyfin.service"
         "docker-feishin.service"
       ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
+    };
+
+    homelab.traefikRouters.feishin = {
+      rule = "Host(`play.jelly.depeuter.dev`)";
+      port = 9180;
+      extraLabels = {
+        "traefik.tls.options.default.minVersion" = "VersionTLS13";
       };
-      script = ''
-        if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
-          ${pkgs.docker}/bin/docker network create ${networkName}
-        fi
-      '';
     };
 
     virtualisation.oci-containers.containers = {
@@ -145,6 +111,7 @@ in {
         ];
         extraOptions = [
           "--network=${networkName}"
+          "--network=${proxyNet}"
         ];
         environment = {
           # pre defined server name
@@ -157,12 +124,6 @@ in {
           SERVER_URL= "https://jelly.depeuter.dev";
           TZ = config.time.timeZone;
         };
-        labels = {
-          "traefik.enable" = "true";
-          "traefik.http.routers.feishin.rule" = "Host(`play.jelly.depeuter.dev`)";
-          "traefik.http.services.feishin.loadbalancer.server.port" = feishinPort;
-          "traefik.tls.options.default.minVersion" = "VersionTLS13";
-        };
       };
     };
   };

nixos/modules/apps/monitoring/default.nix

@@ -0,0 +1,164 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.homelab.apps.monitoring;
+in {
+  options.homelab.apps.monitoring.enable = lib.mkEnableOption "Homelab Monitoring Stack";
+
+  config = lib.mkIf cfg.enable {
+    sops.secrets."grafana/admin_password" = {
+    sopsFile = ../../../../secrets/prod/monitoring.yaml;
+  };
+  sops.secrets."alertmanager/smtp_password" = {
+    sopsFile = ../../../../secrets/prod/monitoring.yaml;
+  };
+
+  # 1. Loki Log Storage
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      server.http_listen_port = 3100;
+      common.ring.instance_addr = "127.0.0.1";
+      common.ring.kvstore.store = "inmemory";
+      schema_config = {
+        configs = [{
+          from = "2020-10-24";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+        };
+        filesystem.directory = "/var/lib/loki/chunks";
+      };
+    };
+  };
+
+  # 2. Prometheus Time-Series DB
+  services.prometheus = {
+    enable = true;
+    port = 9090;
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        scrape_interval = "15s";
+        # In a real setup, we would use Prometheus service discovery (e.g., file_sd_configs) 
+        # or list all homelab IPs here. For now, we scrape localhost.
+        static_configs = [{
+          targets = [ "127.0.0.1:9100" ];
+        }];
+      }
+    ];
+    # Connect Prometheus to Alertmanager
+    alertmanagers = [{
+      static_configs = [{
+        targets = [ "127.0.0.1:9093" ];
+      }];
+    }];
+  };
+
+  # 3. Alertmanager (Routing alerts to NTFY and Email)
+  services.prometheus.alertmanager = {
+    enable = true;
+    port = 9093;
+    configuration = {
+      global = {
+        smtp_smarthost = "smtp.example.com:587";
+        smtp_from = "alerts@depeuter.dev";
+        smtp_auth_username = "alerts@depeuter.dev";
+        smtp_auth_password_file = config.sops.secrets."alertmanager/smtp_password".path;
+      };
+      route = {
+        receiver = "ntfy-and-email";
+        group_wait = "30s";
+        group_interval = "5m";
+        repeat_interval = "4h";
+        group_by = [ "alertname" "instance" ];
+      };
+      receivers = [{
+        name = "ntfy-and-email";
+        email_configs = [{
+          to = "your-email@example.com";
+          # Use smarthost settings defined in global
+        }];
+        webhook_configs = [{
+          # Alertmanager natively supports webhooks. We send the JSON to NTFY's Prometheus endpoint.
+          url = "http://127.0.0.1:2586/alerts"; 
+        }];
+      }];
+    };
+  };
+
+  # 4. NTFY Push Notification Server
+  services.ntfy-sh = {
+    enable = true;
+    settings = {
+      base-url = "https://ntfy.lab.depeuter.dev";
+      listen-http = ":2586";
+      # You can configure auth via the CLI once the service is running, or via auth-file.
+      # For now, it is open locally.
+    };
+  };
+
+  # 5. Grafana Visualization
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      http_port = 3000;
+      http_addr = "127.0.0.1";
+      domain = "grafana.lab.depeuter.dev";
+    };
+    settings.security.admin_password = "$__file{${config.sops.secrets."grafana/admin_password".path}}";
+    
+    # Declarative Data Sources
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Prometheus";
+          type = "prometheus";
+          access = "proxy";
+          url = "http://127.0.0.1:9090";
+          isDefault = true;
+        }
+        {
+          name = "Loki";
+          type = "loki";
+          access = "proxy";
+          url = "http://127.0.0.1:3100";
+        }
+      ];
+    };
+  };
+
+  # 6. Traefik Reverse Proxy for Grafana and NTFY
+  homelab.apps.traefik.dynamicConfigOptions.http = {
+    routers = {
+      grafana = {
+        rule = "Host(`grafana.lab.depeuter.dev`)";
+        service = "grafana";
+      };
+      ntfy = {
+        rule = "Host(`ntfy.lab.depeuter.dev`)";
+        service = "ntfy";
+      };
+    };
+    services = {
+      grafana.loadBalancer.servers = [{ url = "http://host.docker.internal:3000"; }];
+      ntfy.loadBalancer.servers = [{ url = "http://host.docker.internal:2586"; }];
+    };
+  };
+
+  # Open firewall for Loki so agents can push logs
+  networking.firewall.allowedTCPPorts = [ 3100 ];
+  };
+}

nixos/modules/apps/solidtime/default.nix

@@ -92,24 +92,13 @@ in {
     homelab.virtualisation.containers.enable = true;
 
     # Make sure the Docker network exists.
-    systemd.services = {
-      "docker-${networkName}-create-network" = {
-        description = "Create Docker network for ${networkName}";
+    homelab.dockerNetworks = {
+      "${networkName}" = {
         requiredBy = [
           "${containers.solidtime.serviceName}.service"
         ];
-        serviceConfig = {
-          Type = "oneshot";
-          RemainAfterExit = true;
-        };
-        script = ''
-          if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
-            ${pkgs.docker}/bin/docker network create ${networkName}
-          fi
-        '';
       };
-      "docker-${internalNetworkName}-create-network" = {
-        description = "Create Docker network for ${internalNetworkName}";
+      "${internalNetworkName}" = {
         requiredBy = [
           "${containers.solidtime.serviceName}.service"
           "${containers.solidtimeScheduler.serviceName}.service"
@@ -117,15 +106,6 @@ in {
           "${containers.solidtimeDb.serviceName}.service"
           "${containers.solidtimeGotenberg.serviceName}.service"
         ];
-        serviceConfig = {
-          Type = "oneshot";
-          RemainAfterExit = true;
-        };
-        script = ''
-          if ! ${pkgs.docker}/bin/docker network ls | grep -q ${internalNetworkName}; then
-            ${pkgs.docker}/bin/docker network create ${internalNetworkName}
-          fi
-        '';
       };
     };
     

nixos/modules/apps/traefik/default.nix

@@ -12,26 +12,21 @@ in {
       default = "traefik";
       description = "The name of the shared network to connect the container to.";
     };
+    dynamicConfigOptions = lib.mkOption {
+      type = lib.types.attrs;
+      default = {};
+      description = "Dynamic configuration options to write to file and mount into Traefik.";
+    };
   };
 
   config = lib.mkIf cfg.enable {
     homelab.virtualisation.containers.enable = true;
 
     # Make sure the Docker network exists.
-    systemd.services."docker-${cfg.sharedNetworkName}-create-network" = {
-      description = "Create Docker network for ${cfg.sharedNetworkName}";
+    homelab.dockerNetworks."${cfg.sharedNetworkName}" = {
       requiredBy = [
         "docker-traefik.service"
       ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-      script = ''
-        if ! ${pkgs.docker}/bin/docker network ls | grep -q ${cfg.sharedNetworkName}; then
-          ${pkgs.docker}/bin/docker network create ${cfg.sharedNetworkName}
-        fi
-      '';
     };
 
     virtualisation.oci-containers.containers.traefik = {
@@ -45,6 +40,7 @@ in {
       ];
       extraOptions = [
         "--network=${cfg.sharedNetworkName}"
+        "--add-host=host.docker.internal:host-gateway"
       ];
       environmentFiles = [
         /home/admin/.cloudflare.secret
@@ -56,6 +52,10 @@ in {
         "--providers.docker=true"
         "--providers.docker.exposedByDefault=false"
 
+        # Add File provider
+        "--providers.file.filename=/etc/traefik/dynamic_conf.yml"
+        "--providers.file.watch=true"
+
         # Add web entrypoint
         "--entrypoints.web.address=:80/tcp"
         "--entrypoints.web.http.redirections.entrypoint.to=websecure"
@@ -75,10 +75,14 @@ in {
         "--certificatesresolvers.letsencrypt.acme.email=tibo.depeuter@telenet.be"
         "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
       ];
-      volumes = [
+      volumes = let
+        dynamicConfFormat = pkgs.formats.yaml { };
+        dynamicConfFile = dynamicConfFormat.generate "traefik-dynamic-conf.yml" cfg.dynamicConfigOptions;
+      in [
         "letsencryp:/letsencrypt"
 
         "/var/run/docker.sock:/var/run/docker.sock:ro"
+        "${dynamicConfFile}:/etc/traefik/dynamic_conf.yml:ro"
       ];
       labels = {
         "traefik.enable" = "true";

nixos/modules/apps/vaultwarden/default.nix

@@ -33,21 +33,11 @@ in {
     };
 
     # Make sure the Docker network exists.
-    systemd.services."docker-${networkName}-create-network" = {
-      description = "Create Docker network for ${networkName}";
+    homelab.dockerNetworks."${networkName}" = {
       requiredBy = [
         "docker-vaultwarden-db.service"
         "docker-vaultwarden.service"
       ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-      script = ''
-        if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
-          ${pkgs.docker}/bin/docker network create ${networkName}
-        fi
-      '';
     };
 
     virtualisation.oci-containers.containers = let

nixos/modules/common/default.nix

@@ -1,8 +1,18 @@
 {
+  imports = [
+    ./docker.nix
+    ./gitops.nix
+    ./monitoring.nix
+    ./nfs.nix
+    ./traefik.nix
+    ./users.nix
+  ];
+
   config = {
     homelab = {
       services.openssh.enable = true;
       users.admin.enable = true;
+      gitops.enable = true;
     };
 
     nix.settings.experimental-features = [

nixos/modules/common/docker.nix

@@ -0,0 +1,38 @@
+{ config, lib, pkgs, ... }:
+
+{
+  options.homelab.dockerNetworks = lib.mkOption {
+    description = "Declarative Docker networks to create before containers start.";
+    default = {};
+    type = lib.types.attrsOf (lib.types.submodule {
+      options = {
+        requiredBy = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          default = [];
+          description = "List of systemd services that require this network (e.g., docker-containerName.service).";
+        };
+      };
+    });
+  };
+
+  config = {
+    systemd.services = lib.mapAttrs' (networkName: cfg:
+      lib.nameValuePair "docker-${networkName}-create-network" {
+        description = "Create Docker network for ${networkName}";
+        requiredBy = cfg.requiredBy;
+        after = [ "network.target" "docker.service" ];
+        requires = [ "docker.service" ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = pkgs.writeShellScript "create-${networkName}-docker-network" ''
+            if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then
+              ${pkgs.docker}/bin/docker network create ${networkName}
+            fi
+          '';
+        };
+      }
+    ) config.homelab.dockerNetworks;
+  };
+}

nixos/modules/common/gitops.nix

@@ -0,0 +1,103 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.homelab.gitops;
+
+  updateScript = pkgs.writeShellApplication {
+    name = "homelab-gitops-update";
+    runtimeInputs = [ pkgs.git pkgs.nixos-rebuild pkgs.jq pkgs.coreutils ];
+    text = builtins.readFile ../../../scripts/nixos-sync.sh;
+  };
+
+in {
+  options.homelab.gitops = {
+    enable = lib.mkEnableOption "Custom GitOps native deployment system";
+    
+    repoUrl = lib.mkOption {
+      type = lib.types.str;
+      default = "https://git.depeuter.dev/Bos55/nix-config.git";
+      description = "The repository URL to pull configurations from.";
+    };
+    
+    branch = lib.mkOption {
+      type = lib.types.str;
+      default = "v2";
+      description = "The branch to deploy.";
+    };
+
+    useBuilder = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Whether to use the central Builder host to compile packages.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.homelab-gitops = {
+      description = "Homelab GitOps Update Service";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${updateScript}/bin/homelab-gitops-update ${cfg.repoUrl} ${cfg.branch}";
+        # Must run as root to rebuild the system
+        User = "root";
+      };
+    };
+
+    systemd.timers.homelab-gitops = {
+      description = "Timer for Homelab GitOps Update Service";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnBootSec = "5m";
+        OnUnitActiveSec = "5m";
+        RandomizedDelaySec = "30s";
+      };
+    };
+
+    sops.secrets."webhook-secret" = {};
+
+    services.webhook = {
+      enable = true;
+      port = 9000;
+      hooks = {
+        gitops = {
+          execute-command = "${pkgs.systemd}/bin/systemctl";
+          pass-arguments-to-command = [
+            { source = "string"; name = "start"; }
+            { source = "string"; name = "homelab-gitops.service"; }
+          ];
+          trigger-rule = {
+            match = {
+              type = "payload-hash-sha256";
+              secret = "{{ getenv \"WEBHOOK_SECRET\" }}";
+              parameter = {
+                source = "header";
+                name = "X-Forgejo-Signature";
+              };
+            };
+          };
+        };
+      };
+    };
+
+    # Inject the secret as an environment variable into the webhook service
+    systemd.services.webhook.serviceConfig.EnvironmentFile = config.sops.secrets."webhook-secret".path;
+
+    sops.secrets."builder-ssh-key" = lib.mkIf cfg.useBuilder {};
+
+    nix.buildMachines = lib.mkIf cfg.useBuilder [
+      {
+        hostName = "builder.depeuter.dev"; # Must be routable from nodes
+        system = "x86_64-linux";
+        sshUser = "builder";
+        sshKey = config.sops.secrets."builder-ssh-key".path;
+        maxJobs = 4;
+        speedFactor = 2;
+        supportedFeatures = [ "nixos-test" "benchmark" "big-parallel" "kvm" ];
+      }
+    ];
+
+    nix.distributedBuilds = lib.mkIf cfg.useBuilder true;
+  };
+}

nixos/modules/common/monitoring.nix

@@ -0,0 +1,47 @@
+{ config, lib, pkgs, ... }:
+
+{
+  # Prometheus Node Exporter for hardware metrics
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = [ "systemd" ];
+      port = 9100;
+    };
+  };
+
+  # Promtail to ship logs to Loki
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 28183;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        # Use the internal DNS name for the Loki ingress
+        url = "http://loki.lab.depeuter.dev/loki/api/v1/push";
+      }];
+      scrape_configs = [{
+        job_name = "journal";
+        journal = {
+          max_age = "12h";
+          labels = {
+            job = "systemd-journal";
+            host = config.networking.hostName;
+          };
+        };
+        relabel_configs = [{
+          source_labels = [ "__journal__systemd_unit" ];
+          target_label = "unit";
+        }];
+      }];
+    };
+  };
+
+  # Open firewall ports for node-exporter so Prometheus can scrape it
+  networking.firewall.allowedTCPPorts = [ 9100 ];
+}

nixos/modules/common/nfs.nix

@@ -0,0 +1,46 @@
+{ config, lib, ... }:
+
+{
+  options.homelab.nfsMounts = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.submodule {
+      options = {
+        device = lib.mkOption {
+          type = lib.types.str;
+          description = "The NFS device, e.g. 192.168.0.11:/mnt/POOL/DATA";
+        };
+        readOnly = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = "Whether to mount the NFS share read-only";
+        };
+        extraOptions = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          default = [
+            "async"
+            "soft"
+            "timeo=100"
+            "retry=50"
+            "actimeo=1800"
+            "lookupcache=all"
+          ];
+          description = "Extra NFS mount options to append";
+        };
+      };
+    });
+    default = {};
+    description = "NFS mounts to automatically configure with standard homelab options";
+  };
+
+  config = {
+    fileSystems = lib.mapAttrs (path: cfg: {
+      device = cfg.device;
+      fsType = "nfs";
+      options = [
+        "nfsvers=4.2"
+        "nosuid"
+        "tcp"
+      ] ++ (if cfg.readOnly then [ "ro" ] else [ "rw" ])
+        ++ cfg.extraOptions;
+    }) config.homelab.nfsMounts;
+  };
+}

nixos/modules/common/traefik.nix

@@ -0,0 +1,65 @@
+{ config, lib, ... }:
+
+let
+  proxyNet = config.homelab.apps.traefik.sharedNetworkName;
+in {
+  options.homelab.traefikRouters = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.submodule {
+      options = {
+        rule = lib.mkOption {
+          type = lib.types.str;
+          description = "The Traefik router rule, e.g. Host(`example.com`)";
+        };
+        port = lib.mkOption {
+          type = lib.types.int;
+          description = "The port the service listens on";
+        };
+        middlewares = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          default = [];
+          description = "Middlewares to apply";
+        };
+        tls = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = "Whether to enable TLS";
+        };
+        entryPoints = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          default = [];
+          description = "Entrypoints to use";
+        };
+        extraLabels = lib.mkOption {
+          type = lib.types.attrsOf lib.types.str;
+          default = {};
+          description = "Extra labels to apply";
+        };
+      };
+    });
+    default = {};
+    description = "Declarative Traefik router configuration";
+  };
+
+  config = {
+    # Generate labels for containers based on homelab.traefikRouters
+    # This assumes that the name in homelab.traefikRouters matches the container name.
+    virtualisation.oci-containers.containers = lib.mapAttrs (name: router: {
+      labels = {
+        "traefik.enable" = "true";
+        "traefik.docker.network" = proxyNet;
+        "traefik.http.routers.${name}.rule" = router.rule;
+        "traefik.http.services.${name}.loadbalancer.server.port" = toString router.port;
+      }
+      // lib.optionalAttrs (router.middlewares != []) {
+        "traefik.http.routers.${name}.middlewares" = builtins.concatStringsSep "," router.middlewares;
+      }
+      // lib.optionalAttrs router.tls {
+        "traefik.http.routers.${name}.tls" = "true";
+      }
+      // lib.optionalAttrs (router.entryPoints != []) {
+        "traefik.http.routers.${name}.entryPoints" = builtins.concatStringsSep "," router.entryPoints;
+      }
+      // router.extraLabels;
+    }) config.homelab.traefikRouters;
+  };
+}

nixos/modules/common/users.nix

@@ -0,0 +1,37 @@
+{ config, lib, ... }:
+
+{
+  options.homelab.appUsers = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.submodule {
+      options = {
+        uid = lib.mkOption {
+          type = lib.types.int;
+          description = "The user ID for the app user";
+        };
+        group = lib.mkOption {
+          type = lib.types.str;
+          default = config.users.groups.apps.name;
+          description = "The primary group for the app user";
+        };
+        extraGroups = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          default = [];
+          description = "Extra groups for the app user";
+        };
+      };
+    });
+    default = {};
+    description = "App users to automatically create with standard homelab options";
+  };
+
+  config = {
+    users.users = lib.mapAttrs (name: cfg: {
+      uid = lib.mkForce cfg.uid;
+      isSystemUser = true;
+      group = cfg.group;
+      extraGroups = cfg.extraGroups;
+      home = "/var/empty";
+      shell = null;
+    }) config.homelab.appUsers;
+  };
+}

nixos/modules/default.nix

@@ -1,10 +1,9 @@
 {
   imports = [
     ./apps
+    ./common
     ./fileSystems
     ./services
     ./virtualisation
-
-    ./common
   ];
 }

nixos/modules/fileSystems/media/video/default.nix

@@ -25,17 +25,14 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
-    fileSystems."${cfg.hostPath}" = {
+    homelab.nfsMounts."${cfg.hostPath}" = {
       device = "192.168.0.11:${remotePath}";
-      fsType = "nfs";
-      options = [
-        permissionsOption
+      readOnly = permissionsOption == "ro";
+      extraOptions = [
         "auto"
-        "nfsvers=4.2"
-        "async" "soft"
         "rsize=1048576" "wsize=1048576"
-        "timeo=600" "retry=50" "retrans=2" "actimeo=1800" "lookupcache=all"
-        "_netdev" "nosuid" "tcp"
+        "timeo=600" "retrans=2"
+        "_netdev"
       ];
     };
   };

nixos/modules/services/default.nix

@@ -2,5 +2,6 @@
   imports = [
     ./actions
     ./openssh
+    ./hypervisor-gitops
   ];
 }

nixos/modules/services/hypervisor-gitops/default.nix

@@ -0,0 +1,118 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.homelab.services.hypervisor-gitops;
+  
+  hypervisorSyncScript = pkgs.writeShellApplication {
+    name = "hypervisor-sync";
+    runtimeInputs = with pkgs; [ git opentofu coreutils ];
+    text = builtins.readFile ../../../../scripts/hypervisor-sync.sh;
+  };
+
+  stagingSyncScript = pkgs.writeShellApplication {
+    name = "staging-sync";
+    runtimeInputs = with pkgs; [ opentofu coreutils jq curl ];
+    text = builtins.readFile ../../../../scripts/staging-sync.sh;
+  };
+  
+  natsConsumerScript = pkgs.writeShellApplication {
+    name = "nats-consumer";
+    runtimeInputs = with pkgs; [ natscli jq stagingSyncScript ];
+    text = ''
+      set -euo pipefail
+      
+      NATS_URL=''${NATS_URL:-"nats://192.168.0.20:4222"}
+      
+      echo "Starting NATS JetStream consumer for staging env..."
+      
+      # Try to create stream and consumer if they don't exist
+      nats --server "$NATS_URL" stream add FORGEJO_EVENTS --subjects "forgejo.staging" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file -f || true
+      nats --server "$NATS_URL" consumer add FORGEJO_EVENTS STAGING --pull --ack explicit --filter forgejo.staging --deliver all -f || true
+      
+      echo "Listening for messages..."
+      while true; do
+        # We use a simple sub to pull messages. In a real environment, 
+        # a dedicated Go/Python client is better for manual explicit acks.
+        # This will auto-ack upon receipt and pass to the staging script.
+        nats --server "$NATS_URL" sub "forgejo.staging" | awk '/\[#.*\]/{flag=1; next} flag' | staging-sync || true
+      done
+    '';
+  };
+in {
+  options.homelab.services.hypervisor-gitops = {
+    enable = mkEnableOption "Hypervisor GitOps Service";
+    
+    repoUrl = mkOption {
+      type = types.str;
+      description = "The URL of the git repository to pull";
+    };
+
+    # TODO Replace with webhooks
+    pollInterval = mkOption {
+      type = types.str;
+      default = "hourly";
+      description = "Systemd calendar event for polling interval";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [
+      git
+      opentofu
+      natscli
+      jq
+    ];
+
+    systemd.services.hypervisor-gitops = {
+      description = "Hypervisor GitOps Polling Service";
+      
+      # We need network access to reach Forgejo and Proxmox API
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      
+      serviceConfig = {
+        Type = "oneshot";
+        User = "root"; # Needs root to read SOPS secrets potentially
+        
+        # We will create a state directory for the repo
+        StateDirectory = "hypervisor-gitops";
+        WorkingDirectory = "/var/lib/hypervisor-gitops";
+        
+        ExecStart = "${hypervisorSyncScript}/bin/hypervisor-sync ${cfg.repoUrl} opentofu/nodes/mikoshi";
+      };
+    };
+
+    systemd.timers.hypervisor-gitops = {
+      description = "Timer for Hypervisor GitOps Service";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.pollInterval;
+        Persistent = true;
+      };
+    };
+
+    systemd.services.staging-sync = {
+      description = "Staging Environment NATS Consumer";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      
+      serviceConfig = {
+        Type = "simple";
+        User = "root"; # Needs root to read SOPS secrets
+        Restart = "always";
+        RestartSec = "10s";
+        
+        StateDirectory = "hypervisor-gitops";
+        WorkingDirectory = "/var/lib/hypervisor-gitops";
+        
+        # We assume TRUENAS_API_KEY is provided via a sops EnvironmentFile
+        # EnvironmentFile = config.sops.secrets."truenas-api-key".path;
+        
+        ExecStart = "${natsConsumerScript}/bin/nats-consumer";
+      };
+    };
+  };
+}

nixos/users/admin/default.nix

@@ -9,7 +9,7 @@ in {
       type = lib.types.listOf lib.types.str;
       default = [
         # HomeLab > NixOS > admin > ssh
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGWIOOEqTy8cWKpENVbzD4p7bsQgQb/Dgpzk8i0dZ00T"
+        "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIHOoTp+e6qWGn4Sco5CZ6G0zrX5NAQBpLlDVirncJ/HqAAAABHNzaDo="
       ];
     };
   };

opentofu/modules/proxmox-node/main.tf

@@ -0,0 +1,122 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "~> 0.61.0"
+    }
+  }
+}
+
+# Bare Metal Host Configurations (SSH Provisioning)
+
+# We use a null_resource to run imperative commands on the Debian host
+# that are not currently supported by the bpg/proxmox provider.
+resource "null_resource" "bare_metal_setup" {
+  triggers = {
+    node         = var.node_name
+    data_disk_id = var.data_disk_id
+    pool_name    = var.zfs_pool_name
+  }
+
+  connection {
+    type     = "ssh"
+    user     = "root"
+    # Assuming running locally on the node during bootstrap, or via SSH if run from a laptop.
+    # We default to local host if run from Control Center, but for flexibility we use the endpoint.
+    host     = var.node_name 
+    agent    = true
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      # Set laptop lid switch to ignore (prevents sleeping when closed)
+      "sed -i 's/^#\\?HandleLidSwitch=.*/HandleLidSwitch=ignore/' /etc/systemd/logind.conf",
+      "systemctl restart systemd-logind",
+      
+      # Format the ZFS pool if it doesn't already exist
+      "zpool list ${var.zfs_pool_name} || zpool create -f ${var.zfs_pool_name} /dev/disk/by-id/${var.data_disk_id}",
+      
+      # Register the ZFS pool in Proxmox if it's not already registered
+      "pvesm status -storage ${var.zfs_pool_name} || pvesm add zfspool ${var.zfs_pool_name} --pool ${var.zfs_pool_name} --content images,rootdir"
+    ]
+  }
+}
+
+# Resource Pools
+
+# Creates logical groups for organizing and securing VMs.
+resource "proxmox_virtual_environment_pool" "core" {
+  pool_id = "core"
+  comment = "Critical, public-facing services (reverse proxy, DNS)"
+}
+
+resource "proxmox_virtual_environment_pool" "production" {
+  pool_id = "production"
+  comment = "Stable services with backup strategies"
+}
+
+resource "proxmox_virtual_environment_pool" "dev" {
+  pool_id = "dev"
+  comment = "Persistent but non-production services, run 24/7"
+}
+
+resource "proxmox_virtual_environment_pool" "staging" {
+  pool_id = "staging"
+  comment = "Ephemeral test VMs managed by Pull-Based PR webhooks"
+}
+
+# API Tokens for Control Center
+
+# Creates a restricted user for the Control Center VM to manage the cluster.
+resource "proxmox_virtual_environment_role" "control_center_role" {
+  role_id = "ControlCenter"
+
+  privileges = [
+    "VM.Allocate",
+    "VM.Audit",
+    "VM.Clone",
+    "VM.Config.CDROM",
+    "VM.Config.CPU",
+    "VM.Config.Disk",
+    "VM.Config.HWType",
+    "VM.Config.Memory",
+    "VM.Config.Network",
+    "VM.Config.Options",
+    "VM.Monitor",
+    "VM.PowerMgmt",
+    "Datastore.AllocateSpace",
+    "Datastore.Audit",
+    "SDN.Use"
+  ]
+}
+
+resource "proxmox_virtual_environment_user" "control_center_user" {
+  user_id = "control-center@pve"
+  comment = "Managed by OpenTofu - Used by Control Center for GitOps"
+}
+
+# Grant full VM access to the control center
+resource "proxmox_virtual_environment_acl" "control_center_vms" {
+  user_id = proxmox_virtual_environment_user.control_center_user.user_id
+  role_id = proxmox_virtual_environment_role.control_center_role.role_id
+  path    = "/vms"
+}
+
+# Grant datastore access
+resource "proxmox_virtual_environment_acl" "control_center_storage" {
+  user_id = proxmox_virtual_environment_user.control_center_user.user_id
+  role_id = proxmox_virtual_environment_role.control_center_role.role_id
+  path    = "/storage/${var.zfs_pool_name}"
+}
+
+resource "proxmox_virtual_environment_user_token" "control_center_token" {
+  user_id    = proxmox_virtual_environment_user.control_center_user.user_id
+  token_name = "gitops"
+  comment    = "Stored only on the Control Center VM"
+}
+
+output "control_center_api_token" {
+  value       = proxmox_virtual_environment_user_token.control_center_token.value
+  sensitive   = true
+  description = "The secret API token for control-center@pve. This will be injected into the Control Center SOPS config."
+}

opentofu/modules/proxmox-node/variables.tf

@@ -0,0 +1,15 @@
+variable "node_name" {
+  type        = string
+  description = "The name of the Proxmox node (e.g. pve)"
+}
+
+variable "data_disk_id" {
+  type        = string
+  description = "The persistent block device ID for the data disk (e.g. wwn-0x500...)"
+}
+
+variable "zfs_pool_name" {
+  type        = string
+  default     = "data"
+  description = "The name of the ZFS pool to create on the data disk"
+}

opentofu/nodes/mikoshi/main.tf

@@ -0,0 +1,81 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "~> 0.61.0"
+    }
+  }
+}
+
+provider "proxmox" {
+  # Endpoint and credentials will be picked up from environment variables 
+  # or passed via the bootstrap script.
+  # PROXMOX_VE_ENDPOINT
+  # PROXMOX_VE_USERNAME 
+  # PROXMOX_VE_PASSWORD
+  # PROXMOX_VE_INSECURE=true
+}
+
+module "proxmox_node" {
+  source = "../../modules/proxmox-node"
+  node_name = "mikoshi"
+  data_disk_id = "nvme-KXG80ZNV2T04_NVMe_KIOXIA_2048GB_241C11Y5EHAK"
+  zfs_pool_name = "data"
+}
+
+output "control_center_api_token" {
+  value       = module.proxmox_node.control_center_api_token
+  sensitive   = true
+  description = "The secret API token for control-center@pve."
+}
+
+resource "proxmox_virtual_environment_vm" "control_center" {
+  depends_on = [module.proxmox_node]
+  
+  name        = "izanagi"
+  description = "Managed by OpenTofu - GitOps Control Center"
+  tags        = ["infrastructure", "gitops"]
+  node_name   = "mikoshi"
+  vm_id       = 100001000
+
+  on_boot = true
+  
+  pool_id = "core"
+
+  cpu {
+    cores = 2
+    type  = "x86-64-v2-AES"
+  }
+
+  memory {
+    dedicated = 2048
+  }
+
+  agent {
+    enabled = true
+  }
+
+  network_device {
+    bridge = "vmbr0"
+  }
+
+  clone {
+    vm_id = 9000
+    full  = true
+  }
+
+  # Cloud-Init for initial SSH access and SOPS age key injection
+  initialization {
+    ip_config {
+      ipv4 {
+        address = "dhcp"
+      }
+    }
+    user_account {
+      username = "gh0st"
+      keys     = [
+        "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIHOoTp+e6qWGn4Sco5CZ6G0zrX5NAQBpLlDVirncJ/HqAAAABHNzaDo="
+      ]
+    }
+  }
+}

opentofu/staging-env/main.tf

@@ -0,0 +1,131 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "~> 0.61.0"
+    }
+    truenas = {
+      source  = "deevus/truenas"
+      version = "~> 0.1.0"
+    }
+  }
+}
+
+variable "vm_id" {
+  description = "The ID of the VM to create (should be unique per PR)"
+  type        = number
+}
+
+variable "pr_number" {
+  description = "The Pull Request number for this staging environment"
+  type        = string
+}
+
+variable "truenas_snapshot_id" {
+  description = "The ID of the TrueNAS snapshot to clone"
+  type        = string
+}
+
+provider "proxmox" {
+  # Relies on PROXMOX_VE_ENDPOINT and PROXMOX_VE_API_TOKEN environment variables
+}
+
+provider "truenas" {
+  # Relies on TRUENAS_API_KEY and TRUENAS_BASE_URL environment variables
+}
+
+resource "truenas_dataset" "staging_clone" {
+  pool        = "tank"
+  path        = "production/staging-pr-${var.pr_number}"
+  snapshot_id = var.truenas_snapshot_id
+}
+
+resource "proxmox_virtual_environment_vm" "staging_vm" {
+  name        = "staging-pr-${var.pr_number}"
+  description = "Ephemeral staging environment for PR #${var.pr_number}"
+  node_name   = "pve"
+  vm_id       = var.vm_id
+
+  # Clone from the latest golden image template
+  clone {
+    vm_id = 9000
+    full  = true
+  }
+
+  agent {
+    enabled = true
+  }
+
+  cpu {
+    cores = 2
+  }
+
+  memory {
+    dedicated = 2048
+  }
+
+  network_device {
+    bridge   = "vmbr0"
+    firewall = true
+  }
+
+  # Cloud-Init configuration to inject the staging age key and set up networking
+
+    initialization {
+    ip_config {
+      ipv4 {
+        address = "dhcp"
+      }
+    }
+
+    user_data_file_id = "local:snippets/staging-key.yaml"
+  }
+}
+
+
+
+resource "proxmox_virtual_environment_firewall_options" "staging_vm_fw_options" {
+  vm_id      = proxmox_virtual_environment_vm.staging_vm.vm_id
+  node_name  = proxmox_virtual_environment_vm.staging_vm.node_name
+  enable     = true
+  policy_in  = "ACCEPT" 
+  policy_out = "DROP"
+}
+
+resource "proxmox_virtual_environment_firewall_rules" "staging_vm_rules" {
+  vm_id     = proxmox_virtual_environment_vm.staging_vm.vm_id
+  node_name = proxmox_virtual_environment_vm.staging_vm.node_name
+
+  rule {
+    action  = "ACCEPT"
+    type    = "out"
+    dest    = "192.168.0.11"
+    comment = "Allow traffic to TrueNAS"
+  }
+
+  rule {
+    action  = "ACCEPT"
+    type    = "out"
+    dest    = "192.168.0.1"
+    comment = "Allow traffic to Gateway/DNS"
+  }
+
+  rule {
+    action  = "DROP"
+    type    = "out"
+    dest    = "192.168.0.0/24"
+    comment = "Drop traffic to local homelab"
+  }
+
+  rule {
+    action  = "ACCEPT"
+    type    = "out"
+    dest    = "0.0.0.0/0"
+    comment = "Allow outbound internet traffic"
+  }
+}
+
+output "staging_vm_ip" {
+  value       = proxmox_virtual_environment_vm.staging_vm.ipv4_addresses[1][0] # Adjust index based on actual returned interfaces
+  description = "The IP address of the newly spun up staging VM."
+}

scripts/bootstrap.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BANNER="==========================================================="
+printf "%s\n Proxmox Bootstrap\n%s\n\n" "$BANNER" "$BANNER"
+
+echo "Applying post-pve-install fixes (fixing repos)..."
+# Remove enterprise repos and add non-subscription repos safely
+rm -f /etc/apt/sources.list.d/pve-enterprise.list
+
+echo "deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list
+
+# Disable the "No Valid Subscription" nag screen
+sed -i.bak "s/data.status !== 'Active'/false/g" /usr/share/javascript/proxmox-widget-toolkit/proxmoxlib.js
+
+echo ""
+echo "Applying NIC offloading fixes..."
+# The community script disables TSO, GSO, and GRO on physical interfaces.
+# We create a systemd service to ensure this applies on boot.
+cat <<'EOF' > /etc/systemd/system/nic-offload-fix.service
+[Unit]
+Description=Disable NIC offloading (TSO/GRO/GSO) for physical interfaces
+After=network-online.target
+
+[Service]
+Type=oneshot
+# Iterate over all physical interfaces (excluding lo, bridges, veth, etc.)
+ExecStart=/bin/bash -c 'for dev in /sys/class/net/*; do if [ "$(basename "$dev")" != "lo" ] && [[ ! "$(basename "$dev")" =~ ^(vmbr|veth|fwbr|tap|bonding) ]]; then /usr/sbin/ethtool -K "$(basename "$dev")" tso off gso off gro off || true; fi; done'
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+systemctl enable --now nic-offload-fix.service || true
+
+echo ""
+echo "Updating system and installing OpenTofu..."
+apt-get update
+# Install curl, git, gnupg, ethtool, and required apt dependencies
+apt-get install -y apt-transport-https ca-certificates curl git gnupg ethtool
+
+# Install OpenTofu repository and binary
+install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://get.opentofu.org/opentofu.gpg | tee /etc/apt/keyrings/opentofu.gpg >/dev/null
+curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu-repo.gpg >/dev/null
+chmod a+r /etc/apt/keyrings/opentofu.gpg /etc/apt/keyrings/opentofu-repo.gpg
+
+printf "deb [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main\ndeb-src [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main\n" > /etc/apt/sources.list.d/opentofu.list
+chmod a+r /etc/apt/sources.list.d/opentofu.list
+
+apt-get update
+apt-get install -y tofu
+
+echo ""
+echo "Cloning the infrastructure repository..."
+cd /root
+if [ -d "nix-config" ]; then
+    echo "Repository already exists. Pulling latest..."
+    cd nix-config
+    git pull
+else
+    git clone -b v2 https://git.depeuter.dev/Bos55/nix-config.git
+    cd nix-config
+fi
+
+echo ""
+echo "Bootstrapping Proxmox Host State..."
+cd opentofu/nodes/mikoshi
+
+echo "Initializing OpenTofu..."
+tofu init -upgrade
+
+echo "Applying bare-metal state..."
+tofu apply -auto-approve
+
+printf "\n%s\n Bootstrap Complete!\n The ZFS pool, Resource Pools, and the Control Center\n VM have been provisioned.\n\n The Control Center is booting now. Once online, it will automatically\n pull this repository and provision the rest of your VMs!\n%s\n" "$BANNER" "$BANNER"

scripts/hypervisor-sync.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# This script pulls the latest changes from the Git repository 
+# and runs OpenTofu to provision the hypervisor state.
+# Usage: ./hypervisor-sync.sh <REPO_URL> <OPENTOFU_DIR>
+
+REPO_URL=${1:-"https://git.depeuter.dev/Bos55/nix-config.git"}
+OPENTOFU_DIR=${2:-"opentofu/nodes/mikoshi"}
+
+echo "Starting Hypervisor GitOps sync..."
+
+if [ ! -d "nix-config" ]; then
+  echo "Cloning repository from $REPO_URL..."
+  git clone "$REPO_URL" nix-config
+fi
+
+cd nix-config || exit
+
+git fetch origin v2
+
+LOCAL=$(git rev-parse HEAD)
+REMOTE=$(git rev-parse origin/v2)
+
+if [ "$LOCAL" = "$REMOTE" ]; then
+  echo "Already up to date. Nothing to do."
+  exit 0
+fi
+
+echo "Changes detected. Updating from $LOCAL to $REMOTE..."
+git reset --hard origin/v2
+
+echo "Applying OpenTofu changes in $OPENTOFU_DIR..."
+cd "$OPENTOFU_DIR" || exit
+
+tofu init -upgrade
+
+tofu apply -auto-approve
+
+echo "Hypervisor GitOps sync completed successfully."

scripts/nixos-sync.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# This script checks the remote Git repository for changes 
+# and triggers a nixos-rebuild if a new commit is found.
+# Usage: ./nixos-sync.sh <REPO_URL> <BRANCH>
+
+set -euo pipefail
+
+REPO_URL=${1:-"https://git.depeuter.dev/Bos55/nix-config.git"}
+BRANCH=${2:-"main"}
+
+echo "Checking remote hash for $REPO_URL branch $BRANCH..."
+
+# Fetch remote hash, fallback to unknown if it fails
+REMOTE_HASH=$(git ls-remote "$REPO_URL" "refs/heads/$BRANCH" | awk '{print $1}' || true)
+
+if [ -z "$REMOTE_HASH" ]; then
+  echo "WARNING: Could not fetch remote hash. Forcing rebuild to be safe."
+  REMOTE_HASH="unknown_remote"
+fi
+
+LOCAL_HASH="unknown_local"
+if [ -f /run/current-system/configurationRevision ]; then
+  LOCAL_HASH=$(cat /run/current-system/configurationRevision)
+fi
+
+echo "Remote hash: $REMOTE_HASH"
+echo "Local hash:  $LOCAL_HASH"
+
+if [ "$REMOTE_HASH" = "$LOCAL_HASH" ] && [ "$REMOTE_HASH" != "unknown_remote" ] && [ "$LOCAL_HASH" != "unknown" ]; then
+  echo "Hashes match. No update needed."
+  exit 0
+fi
+
+echo "Hashes differ or unknown. Triggering nixos-rebuild..."
+
+# Trigger the build and switch
+nixos-rebuild switch --flake "git+$REPO_URL?dir=nixos&ref=$BRANCH"
+
+echo "Update successful."

scripts/staging-sync.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# This script is triggered by the NATS JetStream consumer on the Control Center.
+# It reads a Forgejo webhook JSON payload from STDIN and orchestrates the
+# Staging VM OpenTofu lifecycle.
+
+set -euo pipefail
+
+# Read JSON payload from STDIN
+PAYLOAD=$(cat)
+
+# Extract fields using jq
+ACTION=$(echo "$PAYLOAD" | jq -r '.action // empty')
+PR_NUMBER=$(echo "$PAYLOAD" | jq -r '.pull_request.number // empty')
+
+if [ -z "$ACTION" ] || [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" == "null" ]; then
+    printf "Invalid or missing action/pr_number in payload. Exiting.\n"
+    exit 0
+fi
+
+# We only care about PR events
+if [[ "$ACTION" != "opened" && "$ACTION" != "reopened" && "$ACTION" != "synchronized" && "$ACTION" != "closed" ]]; then
+    printf "Ignoring PR action: %s\n" "$ACTION"
+    exit 0
+fi
+
+# Ensure workspace directory exists for this PR
+WORKSPACE="/var/lib/hypervisor-gitops/staging-pr-${PR_NUMBER}"
+OPENTOFU_SRC="/var/lib/hypervisor-gitops/nix-config/opentofu/staging-env"
+
+# TrueNAS API variables
+TRUENAS_URL=${TRUENAS_URL:-"https://192.168.0.11"}
+TRUENAS_API_KEY=${TRUENAS_API_KEY:-""}
+ZFS_DATASET=${ZFS_DATASET:-"tank/production"}
+
+get_latest_snapshot() {
+    # Fetch the latest snapshot for the dataset from TrueNAS API
+    # Assumes TRUENAS_API_KEY is exported in the environment by SOPS
+    curl -sS -k -X GET \
+        -H "Authorization: Bearer ${TRUENAS_API_KEY}" \
+        -H "Accept: application/json" \
+        "${TRUENAS_URL}/api/v2.0/zfs/snapshot?id~=${ZFS_DATASET}%25&limit=1&sort=-creation" | jq -r '.[0].id'
+}
+
+printf "Processing PR #%s (Action: %s)\n" "$PR_NUMBER" "$ACTION"
+
+if [[ "$ACTION" == "closed" ]]; then
+    if [ ! -d "$WORKSPACE" ]; then
+        printf "Workspace %s does not exist. Nothing to destroy.\n" "$WORKSPACE"
+        exit 0
+    fi
+    printf "Destroying Staging Environment for PR #%s...\n" "$PR_NUMBER"
+    cd "$WORKSPACE"
+    tofu destroy -var="pr_number=${PR_NUMBER}" -auto-approve
+    
+    # Cleanup
+    cd /
+    rm -rf "$WORKSPACE"
+    printf "Staging Environment Destroyed.\n"
+    exit 0
+fi
+
+# Provisioning (opened, reopened, synchronized)
+printf "Setting up Staging Environment for PR #%s...\n" "$PR_NUMBER"
+
+if [ ! -d "$WORKSPACE" ]; then
+    mkdir -p "$WORKSPACE"
+    cp -r "$OPENTOFU_SRC"/* "$WORKSPACE"/
+fi
+
+cd "$WORKSPACE"
+
+# Fetch latest TrueNAS snapshot dynamically
+LATEST_SNAPSHOT=$(get_latest_snapshot)
+
+if [ -z "$LATEST_SNAPSHOT" ] || [ "$LATEST_SNAPSHOT" == "null" ]; then
+    printf "Failed to retrieve the latest TrueNAS snapshot. Aborting.\n"
+    exit 1
+fi
+
+printf "Latest Snapshot ID: %s\n" "$LATEST_SNAPSHOT"
+
+# Apply OpenTofu
+tofu init -upgrade
+tofu apply -var="pr_number=${PR_NUMBER}" -var="truenas_snapshot_id=${LATEST_SNAPSHOT}" -auto-approve
+
+printf "Staging Environment Provisioned successfully.\n"

scripts/truenas-rbac-setup.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script automates the creation of a restricted TrueNAS user (forgejo-ci)
+# and assigns it a custom privilege role strictly limited to ZFS cloning/snapshots.
+
+echo "=========================================="
+echo " TrueNAS RBAC Setup for CI/CD"
+echo "=========================================="
+echo "This script will create a custom Privilege Role and a Restricted User."
+echo ""
+
+read -p "Enter your TrueNAS IP (e.g., 192.168.0.11): " TRUENAS_IP
+read -s -p "Enter your current TrueNAS Admin Token (root): " ADMIN_TOKEN
+echo ""
+read -p "Enter a password for the new 'forgejo-ci' user: " CI_PASSWORD
+
+BASE_URL="http://${TRUENAS_IP}/api/v2.0"
+HEADERS=(
+    "-H" "Authorization: Bearer ${ADMIN_TOKEN}"
+    "-H" "Content-Type: application/json"
+)
+
+echo ""
+echo "1. Creating Custom Privilege (ci-runner-role)..."
+# In TrueNAS SCALE, we create a privilege that allows specific methods
+PRIV_PAYLOAD=$(cat <<EOF
+{
+  "name": "ci-runner-role",
+  "allowlist": [
+    {"method": "zfs.snapshot.create"},
+    {"method": "zfs.snapshot.clone"},
+    {"method": "zfs.dataset.delete"},
+    {"method": "zfs.snapshot.delete"},
+    {"method": "zfs.snapshot.query"},
+    {"method": "zfs.dataset.query"}
+  ]
+}
+EOF
+)
+
+# Attempt to create privilege (ignore if it already exists)
+HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${BASE_URL}/privilege" "${HEADERS[@]}" -d "${PRIV_PAYLOAD}")
+if [ "$HTTP_STATUS" -eq 200 ]; then
+    echo "  -> Privilege created successfully."
+elif [ "$HTTP_STATUS" -eq 409 ] || [ "$HTTP_STATUS" -eq 422 ]; then
+    echo "  -> Privilege already exists or validation failed (code ${HTTP_STATUS}). Skipping."
+else
+    echo "  -> Warning: Privilege creation returned HTTP ${HTTP_STATUS}. (Your TrueNAS version might handle RBAC differently)."
+fi
+
+echo "2. Creating Restricted User (forgejo-ci)..."
+USER_PAYLOAD=$(cat <<EOF
+{
+  "username": "forgejo-ci",
+  "full_name": "Forgejo CI Runner",
+  "password": "${CI_PASSWORD}",
+  "password_disabled": false,
+  "group_create": true,
+  "attributes": {}
+}
+EOF
+)
+
+HTTP_STATUS=$(curl -s -o /tmp/truenas_user.json -w "%{http_code}" -X POST "${BASE_URL}/user" "${HEADERS[@]}" -d "${USER_PAYLOAD}")
+if [ "$HTTP_STATUS" -eq 200 ]; then
+    echo "  -> User created successfully."
+elif [ "$HTTP_STATUS" -eq 409 ] || [ "$HTTP_STATUS" -eq 422 ]; then
+    echo "  -> User already exists. Skipping."
+else
+    echo "  -> Warning: User creation returned HTTP ${HTTP_STATUS}."
+    cat /tmp/truenas_user.json
+fi
+
+echo ""
+echo "=========================================="
+echo " Setup Complete (or mostly complete)!"
+echo "=========================================="
+echo "Because TrueNAS prevents root from generating API tokens for other users,"
+echo "you must complete the final step manually:"
+echo ""
+echo "1. Log into the TrueNAS Web UI at http://${TRUENAS_IP}"
+echo "2. If the script failed to attach the privilege automatically, go to Credentials > Local Users,"
+echo "   edit 'forgejo-ci', and assign it the ZFS roles."
+echo "3. Log in as 'forgejo-ci' (or use the API Keys menu as Admin to generate a key for that user)."
+echo "4. Copy the newly generated token."
+echo "5. Update the TRUENAS_API_KEY secret in your Forgejo repository."
+echo "=========================================="

scripts/truenas-staging-clone.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script creates a ZFS snapshot of a production dataset and clones it for a staging environment.
+# Required environment variables:
+# TRUENAS_IP: The IP address of the TrueNAS scale instance
+# TRUENAS_API_KEY: The API token for TrueNAS
+# POOL_NAME: The name of the ZFS pool (e.g., "tank")
+# SOURCE_DATASET: The name of the production dataset (e.g., "apps/jellyfin")
+# PR_NUMBER: The Pull Request number
+
+if [[ -z "${TRUENAS_IP:-}" || -z "${TRUENAS_API_KEY:-}" || -z "${POOL_NAME:-}" || -z "${SOURCE_DATASET:-}" || -z "${PR_NUMBER:-}" ]]; then
+    echo "Error: Missing required environment variables."
+    exit 1
+fi
+
+BASE_URL="http://${TRUENAS_IP}/api/v2.0"
+HEADERS=(
+    "-H" "Authorization: Bearer ${TRUENAS_API_KEY}"
+    "-H" "Content-Type: application/json"
+)
+
+DATASET_ID="${POOL_NAME}/${SOURCE_DATASET}"
+SNAPSHOT_NAME="pr-${PR_NUMBER}-base"
+CLONE_DATASET_NAME="${SOURCE_DATASET}-pr-${PR_NUMBER}"
+
+echo "1. Creating snapshot of ${DATASET_ID} @ ${SNAPSHOT_NAME}..."
+curl -s -X POST "${BASE_URL}/zfs/snapshot" "${HEADERS[@]}" \
+    -d "{\"dataset\": \"${DATASET_ID}\", \"name\": \"${SNAPSHOT_NAME}\"}" > /dev/null
+
+echo "2. Cloning snapshot to ${POOL_NAME}/${CLONE_DATASET_NAME}..."
+curl -s -X POST "${BASE_URL}/zfs/snapshot/clone" "${HEADERS[@]}" \
+    -d "{\"snapshot\": \"${DATASET_ID}@${SNAPSHOT_NAME}\", \"dataset_dst\": \"${CLONE_DATASET_NAME}\"}" > /dev/null
+
+echo "Staging dataset cloned successfully."

scripts/truenas-staging-teardown.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script destroys the ephemeral staging ZFS clone and the base snapshot.
+# Required environment variables:
+# TRUENAS_IP: The IP address of the TrueNAS scale instance
+# TRUENAS_API_KEY: The API token for TrueNAS
+# POOL_NAME: The name of the ZFS pool (e.g., "tank")
+# SOURCE_DATASET: The name of the production dataset (e.g., "apps/jellyfin")
+# PR_NUMBER: The Pull Request number
+
+if [[ -z "${TRUENAS_IP:-}" || -z "${TRUENAS_API_KEY:-}" || -z "${POOL_NAME:-}" || -z "${SOURCE_DATASET:-}" || -z "${PR_NUMBER:-}" ]]; then
+    echo "Error: Missing required environment variables."
+    exit 1
+fi
+
+BASE_URL="http://${TRUENAS_IP}/api/v2.0"
+HEADERS=(
+    "-H" "Authorization: Bearer ${TRUENAS_API_KEY}"
+    "-H" "Content-Type: application/json"
+)
+
+DATASET_ID="${POOL_NAME}/${SOURCE_DATASET}"
+SNAPSHOT_NAME="pr-${PR_NUMBER}-base"
+CLONE_DATASET_ID="${POOL_NAME}/${SOURCE_DATASET}-pr-${PR_NUMBER}"
+
+# TrueNAS API requires the ID to be URL-encoded for dataset deletion
+# URL encoding helper (replace / with %2F)
+ENCODED_CLONE_ID="${CLONE_DATASET_ID//\//%2F}"
+ENCODED_SNAPSHOT_ID="${DATASET_ID}@${SNAPSHOT_NAME}"
+ENCODED_SNAPSHOT_ID="${ENCODED_SNAPSHOT_ID//\//%2F}"
+
+echo "1. Destroying staging clone ${CLONE_DATASET_ID}..."
+curl -s -X DELETE "${BASE_URL}/zfs/dataset/id/${ENCODED_CLONE_ID}" "${HEADERS[@]}" > /dev/null
+
+echo "2. Destroying base snapshot ${DATASET_ID}@${SNAPSHOT_NAME}..."
+curl -s -X DELETE "${BASE_URL}/zfs/snapshot/id/${ENCODED_SNAPSHOT_ID}" "${HEADERS[@]}" > /dev/null
+
+echo "Staging dataset and snapshot cleaned up successfully."

secrets/secrets.yaml

@@ -1,19 +0,0 @@
-users:
-    admin:
-        authorized_keys:
-            NixOS: ENC[AES256_GCM,data:sj2hkUkWp628KuXp+AnncLdawHpxb9fH1ZHnIisP0x9Tght9+/X2sWHpuMSeqi2i/R8B+Wgte66QkuwAOB0j+oB9N+66EhehmWZlK5hD/22p,iv:z18U+LvAQgPDfBBewE3lJmWZd0NGCPwJIe/h3tupuZc=,tag:ZJar3spO66JbDXygdTHh2w==,type:str]
-sops:
-    age:
-        - recipient: age1qzutny0mqpcccqw6myyfntu6wcskruu9ghzvt6r4te7afkqwnguq05ex37
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjUSt2REk2Mmd0bk9ubjJk
-            dXFiY2JNR1dyZW9qTUdzaWZhY3c3amVwQzA0CkZHNVpZVjhsWXhVQVNaR0xONzhh
-            Y0lQaWNaNmpYYVdrRnZIZUhvUFUzcWMKLS0tIDAvSmF0VmpxcnZEQStXUjNCUE5Z
-            RnA2Lzk2WHFxOEh6dHN0aGhVSVpLTW8KA7IOvGDMBtgo4pe0Sw3Lol243xCDAJ4i
-            PhcJFiUObVRFZN7ISlULnOlTO3pT9jWvvmC5rDZWId3PQ8qjPvnOUg==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-10-04T17:33:22Z"
-    mac: ENC[AES256_GCM,data:I7I7uDFEWfw9+4KROtjHMVhaxYrVK5QmLfFZShSajF0A2Zxu9lg+fDGiMHk40JC5zD31P70QS/ipye1mBGQbCbLEA7uBUhNzZ7G1g58cIXF6vSGmt0fovm0MVSxEJ44r05fx6uT4OJu5BYVxYSlG84gTj9rCFXxxcBJMrh+6yaI=,iv:c1vudsp9bg0Pc2ddRyvWn6Tf0LhqNuEjxG9D4PpHqxs=,tag:K/1PSHhrTdsNPcPmRv/2Ew==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.10.2