diff --git a/.forgejo/workflows/build-golden-image.yml b/.forgejo/workflows/build-golden-image.yml new file mode 100644 index 0000000..fe71f64 --- /dev/null +++ b/.forgejo/workflows/build-golden-image.yml @@ -0,0 +1,42 @@ +name: Build Golden Image + +on: + push: + paths: + - 'hosts/Template/**' + - 'flake.nix' + - 'flake.lock' + workflow_dispatch: + +jobs: + build: + runs-on: self-hosted + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: Build Proxmox Image + run: | + nix run github:nix-community/nixos-generators -- --format proxmox -c hosts/Template/default.nix -o result + + - name: Upload to Proxmox + env: + PROXMOX_URL: "https://proxmox.local:8006/api2/json" + PROXMOX_TOKEN_ID: "terraform@pve!tf-automation" + PROXMOX_TOKEN_SECRET: ${{ secrets.PROXMOX_TOKEN_SECRET }} + NODE_NAME: "pve" + STORAGE_NAME: "local-zfs" + VMID: 9000 + run: | + IMAGE_PATH=$(find result -name "*.qcow2" | head -n 1) + echo "Uploading $IMAGE_PATH to Proxmox as Template $VMID" + # In a real scenario, this would use a script or API client to upload the image + # e.g., using qm importdisk via ssh or the Proxmox API directly. + # For example, using a custom script: ./scripts/upload-to-proxmox.sh $IMAGE_PATH $VMID + echo "TODO: Implement Proxmox upload API call using PROXMOX_TOKEN_SECRET" diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml new file mode 100644 index 0000000..eb11f07 --- /dev/null +++ b/.forgejo/workflows/renovate.yml @@ -0,0 +1,25 @@ +name: RenovateBot + +on: + schedule: + # Run at 2:00 AM every day + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + renovate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Self-hosted Renovate + uses: renovatebot/github-action@v40.1.11 + env: + # Token needs permission to read/write repository contents and pull requests + RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }} + # The platform needs to be explicitly defined for Forgejo/Gitea + RENOVATE_PLATFORM: "gitea" + RENOVATE_ENDPOINT: "https://your-forgejo-instance.local/api/v1" + # The repository format: owner/repo + RENOVATE_REPOSITORIES: "your-username/nix-config" diff --git a/.forgejo/workflows/staging.yml b/.forgejo/workflows/staging.yml new file mode 100644 index 0000000..9ab3470 --- /dev/null +++ b/.forgejo/workflows/staging.yml @@ -0,0 +1,76 @@ +name: Dynamic Staging Environment + +on: + pull_request: + types: [opened, synchronize, closed] + +jobs: + validate-flake: + runs-on: self-hosted + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: Validate Flake + run: nix flake check + + manage-staging: + runs-on: self-hosted + needs: validate-flake + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Provision Staging Environment (Opened/Sync) + if: github.event.action == 'opened' || github.event.action == 'synchronize' + env: + PROXMOX_VE_ENDPOINT: "https://proxmox.local:8006/" + PROXMOX_VE_API_TOKEN: ${{ secrets.PROXMOX_TOKEN_SECRET }} + TF_VAR_pr_number: ${{ github.event.pull_request.number }} + # VM_ID could be dynamically generated or based on PR number (e.g., 8000 + PR_NUMBER) + TF_VAR_vm_id: ${{ format('8{0:03}', github.event.pull_request.number) }} + + # TrueNAS integration (Secrets would need to be added to Forgejo) + TRUENAS_IP: "truenas.local" + TRUENAS_API_KEY: ${{ secrets.TRUENAS_API_KEY }} + POOL_NAME: "tank" + SOURCE_DATASET: "apps/production_data" + run: | + echo "Cloning datasets..." + # ./scripts/truenas-staging-clone.sh + + echo "Applying Terraform for PR ${{ github.event.pull_request.number }}..." + cd terraform/staging-env + terraform init + terraform apply -auto-approve + + - name: Teardown Staging Environment (Closed) + if: github.event.action == 'closed' + env: + PROXMOX_VE_ENDPOINT: "https://proxmox.local:8006/" + PROXMOX_VE_API_TOKEN: ${{ secrets.PROXMOX_TOKEN_SECRET }} + TF_VAR_pr_number: ${{ github.event.pull_request.number }} + TF_VAR_vm_id: ${{ format('8{0:03}', github.event.pull_request.number) }} + + # TrueNAS integration + TRUENAS_IP: "truenas.local" + TRUENAS_API_KEY: ${{ secrets.TRUENAS_API_KEY }} + POOL_NAME: "tank" + SOURCE_DATASET: "apps/production_data" + run: | + echo "Destroying Terraform environment for PR ${{ github.event.pull_request.number }}..." + cd terraform/staging-env + terraform init + terraform destroy -auto-approve + + echo "Tearing down datasets..." + # ./scripts/truenas-staging-teardown.sh diff --git a/.sops.yaml b/.sops.yaml index 02cc451..3bba812 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -1,10 +1,29 @@ +# SOPS configuration for NixOS GitOps Migration (v2) +# See README.md for key management instructions. + keys: - - &tdpeuter_Tibo-NixFatDesk age1fva6s64s884z0q2w7de024sp69ucvqu0pg9shrhhqsn3ewlpjfpsh6md7y - - &tdpeuter_Tibo-NixTop age1qzutny0mqpcccqw6myyfntu6wcskruu9ghzvt6r4te7afkqwnguq05ex37 + # Master Keys (Used for generic/global secrets if needed, but usually we encrypt for specific hosts) + - &prod_master age1pq12tgz8e980yvrsvd6c6ct6fa8y8eq0c8hjkdfvhp9k0phsdpjvewrrj42zsj9xhk2h2q6wmzvfnpjcgrm8x2zyzxjwv6urjpe60w89n9c2c0lw4j44qzr5j6ryxnmw8hysvngnf50s7ykrdrvy0aql9ktqqks2zzn48rd0k2mecw2snug23ydc4u4w6vphgusp3nrux9e8s6dsmf3vg7ngyc4ylgsht9fkmt6j4teq5y2sy9v9lye59ugztr29fw566dsdwu6vk3pw8dmelynkh8rg4fe88y3udntgwdcrzhun0j7kvjgxskvutyjlestwdfue27uqkdp94yhq0pgdxz8zs5tnxdnctzcf6v3yg2zlypesuzf8fatxxtt4564rr6uwdgjnepwv77gmw9erqp4hwtu40x4gxcyvz7pqpk724yd5g44ldh80fj53quhga0re7sqckmr2qfscdunqzu6s3am59t0p7dppw9jq0tfjnq4e8y78ywxhnjyct45g4q4jthn2yqh9gsep2cuuxq6pg3meye7qxs4y598t24su8ky8u25hfcfg9km5j5mf47vq9w8zu7kcaw74rqyq95wxdz84synckuut0ejt0yauzmxcyufjfmvgdx4xphvmf6882av97326hjfvr7qs72umjf6jq5jmhgmjz4vxj3j09ew4xxa3v92dcdqmwa3eshhq939c24l0ymv6lcf33wmcrr4ty869jl2yksjg33sc885q0wvrzn5wkd4ewgjmn660hgfgcs03r40wzugl7u7tssrunrs5du0scu077hc5vee2vxxtqrsgmmvg3h5usg8mcjk4yf8gnx6g6pzy076gv3ysxn6vspp3s6eyk2e39dhpj98r9u656kqa3c2tatkw3uucunru92d4z45yqp8q2wy4py3uyp0n9rsu2dv29xqvhnjh8tw324qc6favu59t7mryvz50d96sq2njfmsdqvqw3fvjxfvvrhldzllfasrlr69qt9ujtdeezycx7axwwc3jt5fhydfq7mcgdt4xuf8v4v6kk7y8fe5xxlyfyuhwlqlvpce0yjr2pnf264m5llgtprkpwa940f4n6ju4kgh0vnstw8lnw99pe9ta06pyccuqkdr6p6dkq3kk7v8mw2pxp0vzex0semjau6cewmtdlluqlkwxx9y6cuf0uaj3262qs640arlw99wfsxgzc3ev3r5fkqvy9v028gtr9mfs7s3yuyrug56qtdx0w76azyt3exfeetuqdxx9dm9k70er2ky9dqyjzfrd0ezesz55h8pqp2pltylyaptt7l9pf92qtj5cye7q2ezk2567v6hwus58rdrxefprcgshscg7ttc4hsact5c57zc6kqretwqs6dxk00qyznfz9uxzuuufekt275mxxcvf8ad4su08u3k5d56gptxcpjsg60d3qm7e4ryyqajxlq6kf4659jhvqv52fvs0ztv9zmrkz3y08hd9snurakf3fth9202gwur8gmrs3g4pdqunmrugk8kttsdxmhyegzwg46vcytuas4q0nsrjxanng6jp896szgtqm6xf5l3v86kyy5lxxtn2ht6j5wy5mkygmwus9fpzkrwqdzht3up28mesr6rfzj3pvepgdyq5nhsqzdrcu2u3xcwnx6ujvght6hgr4qg555dky50uvqxete2xcf4e6udh2d76ulakfe4uv3uxly93p4tpx4fdxgezyq9yxpmk2730akezna9fswmakxzg9446nft0ry2zy2ftnra9lr6m3vx96xszwd44rnrewt0x5fhjj248psmyztt899y28y4h2g5j3veq294np5wqd7vnn27zmkqxzadn5ylj0m3465t4fzzvrmmftdurlcxpxzj0zxmk2nhjw3h7ggtgvgzxe4wasjcfd4xflrzxs7ejp73 + - &staging_master age1w8tg8mpwj2ujxw0p9k36cpgecq285luwl4wf7a5tjtej4t2wffcq0gnzdw + + # Host Keys (Used by the specific VM to decrypt its own secrets at boot) + # - &host_control_center age1... creation_rules: - - path_regex: secrets/[^/]+\.(yaml|json|env|ini)$ + # Staging Environment + # All files under secrets/staging/ are encrypted purely with the staging_master key. + # Ephemeral VMs are injected with this private key at boot via Cloud-Init. + - path_regex: secrets/staging/.*\.ya?ml$ key_groups: - - age: - - *tdpeuter_Tibo-NixFatDesk - - *tdpeuter_Tibo-NixTop + - age: + - *staging_master + + # Production Environment + # All files under secrets/prod/ are encrypted with the prod_master key AND the specific host's key. + # Ephemeral VMs CANNOT decrypt these files. + - path_regex: secrets/prod/.*\.ya?ml$ + key_groups: + - age: + - *prod_master + # Add host keys here as they are provisioned + # - *host_control_center diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md new file mode 100644 index 0000000..b323994 --- /dev/null +++ b/BOOTSTRAP.md @@ -0,0 +1,51 @@ +# NixOS GitOps Bootstrap Guide + +This repository is designed to be fully automated once bootstrapped. We separate concerns into two layers: +1. **Host Layer**: Bare-metal hardware setup on Proxmox. +2. **Workload Layer**: VMs and network resources. + +## 1. Apply the Host Layer + +Before deploying VMs, you need to prepare the Proxmox host. + +1. **Run Proxmox Post-Install Script**: Log into your fresh Proxmox node's shell and run the community `proxmox-ve-helper` post-install script to fix the APT repositories and remove the nag screen: + ```bash + bash -c "$(wget -qLO - https://github.com/tteck/Proxmox/raw/main/misc/post-pve-install.sh)" + ``` +2. Clone this repository to your laptop. +3. Run the host-layer apply script using Docker (requires Docker installed). The script will automatically configure the remaining hardware-specific requirements (like the ZFS pool, NIC offloading for stability, and the laptop lid switch): + ```bash + ./scripts/apply-host-layer.sh + ``` +4. OpenTofu will prompt you for variables like the target `node_ip`, your `ssh_user`, and the `data_disk_id` (e.g. `nvme-eui...`) to format as ZFS. + +## 2. Prepare the Golden Image + +Because the GitOps Control Center must be spun up fully configured without human intervention, you need a pre-built NixOS `.qcow2` image. +1. Build the golden image locally (requires Nix/WSL): + ```bash + nix run github:nix-community/nixos-generators -- --flake ./nixos#izanagi --format qcow + ``` +2. Upload the resulting `.qcow2` file to your Proxmox node. +3. Create a new VM in Proxmox with **ID 9000**. +4. Import the `.qcow2` as its disk and convert the VM into a **Template**. *(Ensure the template has Cloud-Init configured).* + +## 3. Deploy the Workload Layer + +Once the host is prepped and the template exists, you can deploy the base workloads (like the GitOps Control Center). + +1. Change to the workload-layer directory: + ```bash + cd opentofu/workload-layer/production + ``` +2. Initialize and apply: + ```bash + tofu init + tofu apply + ``` + +## Post-Bootstrap + +Once the `tofu apply` completes, the Control Center VM will boot, initialize via Cloud-Init, and automatically start pulling this git repository. + +From this point on, **you no longer need to manually run tofu apply.** All future changes to VMs, networks, or applications should be done declaratively via Pull Requests to your repository! diff --git a/DISASTER_RECOVERY.md b/DISASTER_RECOVERY.md new file mode 100644 index 0000000..2d5464b --- /dev/null +++ b/DISASTER_RECOVERY.md @@ -0,0 +1,32 @@ +# Disaster Recovery Plan (Zero-Login Strategy) + +In the event of a catastrophic failure, this document outlines how to restore services without causing configuration drift (i.e., without SSHing directly into target nodes and making manual undocumented changes). + +## 1. Full Proxmox Host Freeze (Hardware/GPU Bug) +If the entire Proxmox hypervisor freezes (often caused by the AMD GPU reset bug when a VM reboots): +* **DO NOT** attempt to SSH into the host or the VMs. +* **Recovery Action**: Use your Out-of-Band Management (OOBM) solution. Access your PiKVM, IPMI interface, or remotely controlled Smart Plug to perform a **hard power cycle** of the host. +* Upon reboot, Proxmox will automatically start the VMs, and `comin` will pull the latest working configuration. + +## 2. Production Service Failure (Code Bug) +If a bad PR was merged and a stateless service is failing to start: +* **DO NOT** SSH into the VM to fix the config manually. +* **Recovery Action**: Open the Forgejo Web UI and click **Revert** on the merged PR. +* Within 60 seconds, the `comin` agent running on the target VM will detect the new commit on the `v2` branch, pull the reverted configuration, and restart the service automatically. + +## 3. Production Data Corruption (Database/Stateful Bug) +If a bad deployment corrupted persistent data on TrueNAS (e.g., a failed one-way database schema migration): +* **Recovery Action**: + 1. Revert the code PR in Forgejo as described in section 2. + 2. Execute the `scripts/dr-rollback-dataset.sh` script from the secure **Control Center VM**. This script uses the TrueNAS REST API to rollback the specific ZFS dataset to the automated snapshot taken immediately prior to the deployment. + +## 4. Total Git Repository Loss +If the `v2` branch is completely destroyed or the Gitea server is unrecoverable: +* Your `prod-master` private age key is stored safely offline on your USB drive. +* A recent backup of the repository should be available via your external backup mechanism. +* Clone the backup to your local workstation, run the `proxmox-bootstrap` Terraform state locally to ensure the hypervisor is reachable, and use Terraform to redeploy the Control Center VM and Forgejo instances. + +## 5. Manually Obtaining the Golden Image +If the CI pipeline is down and you urgently need to provision a new VM: +1. Run `nix run github:nix-community/nixos-generators -- --format proxmox -c hosts/Template/default.nix -o result` on your local machine. +2. The output `.qcow2` image can be uploaded to the Proxmox Web UI manually under `local` -> `ISO Images` (or directly via `qm importdisk` if you must use SSH as a last resort). diff --git a/NixOS GitOps Migration Specification.md b/NixOS GitOps Migration Specification.md new file mode 100644 index 0000000..878984d --- /dev/null +++ b/NixOS GitOps Migration Specification.md @@ -0,0 +1,138 @@ +# **Infrastructure Specification: Automated NixOS GitOps Environment** + +## **1\. Architecture Overview** + +This specification outlines the migration from a manually deployed NixOS environment to a fully automated, pull-based GitOps architecture. + +* **Source of Truth:** Forgejo repository (v2 branch). +* **Infrastructure Provisioning:** Terraform (via bpg/proxmox provider) executed by Forgejo Actions. +* **Configuration Management:** NixOS Flakes, pulled autonomously by individual nodes via comin. +* **Secret Management:** sops-nix using Age keys (SSH host keys for VM decryption). +* **Storage:** TrueNAS serving persistent data via NFS/iSCSI, utilizing ZFS snapshot cloning for the staging environment. +* **Dependency Management:** RenovateBot running via Forgejo Actions. + +## **2\. Phase 1: Foundation & State Preparation** + +### **2.1 Git Branching Strategy** + +* Initialize a new orphan branch named v2 (or a fresh repository) to ensure a clean slate for the flake lockfile and SOPS configuration. +* Cherry-pick existing .nix service modules from the legacy main branch as they are migrated. + +### **2.2 Secret Management (Bankruptcy & Reset)** + +* Delete all legacy .sops.yaml configurations. +* Generate a new master age key. This key will be stored securely offline (e.g., Bitwarden) and **never** placed on the Forgejo runner or target VMs. +* Initialize a new .sops.yaml. Target VMs will be added via their public SSH host keys as they are provisioned. + +### **2.3 Storage Strategy (TrueNAS & Staging)** + +To provide the Staging VM with production-accurate data without risking corruption: + +1. **Production State:** TrueNAS maintains the primary ZFS datasets. +2. **Staging State (On-Demand):** When the staging environment spins up, a script (via Proxmox/Forgejo Action or TrueNAS API) takes a temporary ZFS snapshot of the production dataset. +3. **Clone & Mount:** The snapshot is cloned and exported via NFS/SMB specifically for the Staging VM. +4. **Teardown:** When Staging is spun down, the ZFS clone is destroyed. + +## **3\. Phase 2: Proxmox Configuration & Least Privilege** + +Terraform requires API access to Proxmox. To adhere to the principle of least privilege, Proxmox RBAC (Role-Based Access Control) will be utilized. + +### **3.1 Proxmox Static Configuration (RBAC)** + +The root user (root@pam) must **never** be used for automation. Instead, a dedicated API user with a highly restricted role must be created. This can be done via the Proxmox Web UI (Datacenter \-\> Permissions) or via the Proxmox shell (pveum). +**1\. Create the Restricted Role (TerraformProv):** +This role grants only the permissions needed to clone templates, configure hardware, and manage power states. +pveum role add TerraformProv \-privs "VM.Allocate VM.Audit VM.Clone VM.Config.CDROM VM.Config.CPU VM.Config.Disk VM.Config.HWType VM.Config.Memory VM.Config.Network VM.Config.Options VM.Monitor VM.PowerMgmt Datastore.AllocateSpace Datastore.Audit SDN.Use" + +**2\. Create the Terraform User:** +Create a user in the Proxmox VE authentication realm (@pve). +pveum user add terraform@pve + +**3\. Assign Permissions to Paths:** +Bind the role to the user, strictly limiting access to the VMs path and the specific storage path where VMs will reside (e.g., local-zfs). +pveum acl modify /vms \-user terraform@pve \-role TerraformProv +pveum acl modify /storage/local-zfs \-user terraform@pve \-role TerraformProv + +**4\. Generate API Token:** +Generate a token for the user. Proxmox will output a Secret ID. This ID is saved into Forgejo Secrets and passed to the Terraform provider. +pveum user token add terraform@pve tf-automation \--privsep 0 + +*(Note: \--privsep 0 ensures the token inherits the user's permissions, rather than requiring separate ACL definitions).* + +## **4\. Phase 3: Infrastructure as Code (Terraform)** + +### **4.1 The Golden Image (.qcow2)** + +A minimal NixOS image is built locally using nixos-generators and uploaded to Proxmox as a template (e.g., ID 9000). This template contains: + +* QEMU Guest Agent. +* Cloud-Init support. +* comin installed. + +### **4.2 Terraform Provisioning** + +Terraform maintains the hardware state. Its responsibilities are strictly limited to: + +* Cloning Template 9000\. +* Allocating vCPU, RAM, and Virtual Disks. +* Using Cloud-Init to inject network configurations (IP, Gateway) and the Forgejo runner's public SSH key for health check access. + +## **5\. Phase 4: CI/CD & Automation** + +### **5.1 Forgejo Runner & Terraform Pipeline** + +* The newly provisioned Forgejo Runner VM registers with Forgejo. +* A workflow (terraform.yml) runs terraform plan on Pull Requests and terraform apply on merges to the main branch. + +### **5.2 GitOps Pull Agent (comin and Alternatives)** + +Once provisioned by Terraform, VMs boot and pull their OS configuration autonomously. + +* **Chosen Tool (comin):** Polling agent designed specifically for NixOS GitOps. Runs as a systemd service, supports multiple git remotes, branch tracking, and local cryptographic verification. +* **Alternative 1 (system.autoUpgrade):** The native NixOS approach utilizing systemd timers. *Rejected* because it lacks advanced Git authentication and webhooks for immediate triggering. +* **Alternative 2 (deploy-rs / colmena):** Excellent push-based deployment tools. *Rejected* because they require the CI runner to maintain SSH root access to the entire fleet, violating our zero-trust/pull-only goals. + +### **5.3 RenovateBot Integration** + +* Renovate is deployed via a Forgejo Action running on a cron schedule. +* Nix flake support is explicitly enabled in renovate.json. +* Renovate parses flake.lock and Docker tags, automatically opening PRs for updates. + +## **6\. Phase 5: Staging Lifecycle & Healthchecks** + +### **6.1 Lifecycle Automation** + +1. **Trigger:** A PR is opened by Renovate or a developer. +2. **Wake-up:** A Forgejo Action calls the Proxmox API to start the nixos-staging VM and triggers the TrueNAS ZFS snapshot clone. +3. **Pull:** The Staging VM boots, connects to the network, and comin pulls the PR's commit hash. + +### **6.2 Automated Testing** + +The Forgejo Action will verify the deployment was successful: + +* **Systemd Checks:** systemctl is-system-running \--wait and systemctl is-active \. +* **Docker Healthchecks:** docker inspect \--format='{{json .State.Health.Status}}' \. +* **HTTP Probes:** Execute curl commands against the Staging VM to verify HTTP 200 OK. + +### **6.3 Spin Down** + +Upon PR merge or closure, the Forgejo Action shuts down the VM via the Proxmox API and destroys the temporary TrueNAS ZFS clone. + +## **7\. Phase 6: Rollbacks & Disaster Recovery** + +Because infrastructure involves both *stateless* configurations (NixOS) and *stateful* data (TrueNAS), a unified rollback strategy is critical. + +### **7.1 Differentiating Failures (Nix vs. TrueNAS)** + +If a deployment fails, the root cause must be identified to determine the rollback path: + +* **Stateless Failure (NixOS/Code):** The service fails to start immediately. System logs (journalctl \-u my-service) show syntax errors, missing binaries, or bad systemd unit definitions. +* **Stateful Failure (TrueNAS/Data):** The service starts but crashes with a database schema version mismatch, "permission denied" on the NFS mount, or missing user data. This means a service (like a Docker container) attempted a one-way database migration on the persistent TrueNAS dataset and failed halfway. + +### **7.2 The Unified Rollback Procedure** + +When a production rollout fails, **do not** attempt to fix it live. + +1. **Revert the State (TrueNAS):** If the failure was stateful (e.g., a bad database migration), immediately log into TrueNAS and rollback the primary dataset to the automated ZFS snapshot taken right before the deployment. +2. **Revert the Code (Forgejo):** Use the Forgejo UI to click "Revert" on the problematic Pull Request. This creates a new commit restoring the previous flake.nix state. +3. **Autonomous Recovery:** Within 60 seconds, the production VM's comin agent will detect the new commit on main, pull the reverted code, apply the old configuration, and reconnect to the restored TrueNAS dataset. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f2b7c8a --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# Bos55 Homelab (NixOS + Proxmox GitOps) + +This repository contains the declarative infrastructure-as-code for the Bos55 Homelab. It leverages NixOS, Flakes, Terraform, and a custom native pull-based GitOps architecture to fully automate the provisioning, configuration, and maintenance of a home server environment. + +## Goals & Philosophy + +The primary goal of this project is **Zero-Login**. +Servers should be treated as cattle, not pets. If a configuration needs to change, it must be changed in this repository. + +* **Declarative Infrastructure:** Every application, user, reverse proxy rule, and network configuration is defined declaratively using custom NixOS modules. +* **Autonomous Pull-Based Deployments:** Nodes autonomously pull updates from this repository via a custom `homelab.gitops` systemd service, securely offloading compilation to a dedicated Build farm. +* **Instant Webhooks:** Merging a Pull Request instantly triggers deployments across the cluster via cryptographic webhooks. +* **Micro-segmented Security:** Ephemeral staging environments are completely isolated from production datasets using Proxmox hypervisor-level firewalls. +* **Secret Management:** Strict separation between production and staging secrets using `sops-nix` and `age`. + +## Quickstart + +If you are setting up this repository from scratch, you **must** perform the initial bootstrap before the automated pipelines can function. + +1. **Bootstrap Keys:** Follow the [BOOTSTRAP.md](./BOOTSTRAP.md) guide to generate your offline `age` master keys and configure Proxmox API access. +2. **Deploy Builder:** Ensure the dedicated Nix `Builder` host is running so other nodes can securely offload package compilation. +3. **Commit & Push:** Make your configuration changes to `flake.nix` or the `hosts/` directory, and push to the `v2` branch. +4. **Autonomous Deployment:** The nodes will automatically fetch the new hash and apply the configuration. + +## Documentation + +* [Bootstrap Guide](./BOOTSTRAP.md) - Initial setup instructions. +* [Disaster Recovery](./DISASTER_RECOVERY.md) - Protocols for handling hardware freezes and ZFS corruption. diff --git a/flake.nix b/flake.nix index 446f4ce..b4bd66e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,23 +1,30 @@ { - description = "Homelab configuration using flakes"; + description = "Homelab configuration using flakes (v2 GitOps)"; inputs = { nixpkgs.url = "nixpkgs/nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; + sops-nix = { url = "github:Mic92/sops-nix"; inputs.nixpkgs.follows = "nixpkgs"; }; + utils = { url = "github:gytis-ivaskevicius/flake-utils-plus"; inputs.flake-utils.follows = "flake-utils"; }; + + pre-commit-hooks = { + url = "github:cachix/pre-commit-hooks.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = inputs@{ self, nixpkgs, - flake-utils, sops-nix, utils, + flake-utils, sops-nix, utils, pre-commit-hooks, ... }: let @@ -26,39 +33,45 @@ utils.lib.mkFlake { inherit self inputs; + outputsBuilder = channels: let + pkgs = channels.nixpkgs; + system = pkgs.system; + pre-commit-check = pre-commit-hooks.lib.${system}.run { + src = ./.; + hooks = { + nixfmt-rfc-style.enable = true; + statix.enable = true; + terraform-format.enable = true; + }; + }; + in { + checks = { + inherit pre-commit-check; + }; + devShells.default = pkgs.mkShell { + inherit (pre-commit-check) shellHook; + buildInputs = pre-commit-check.enabledPackages; + }; + }; + hostDefaults = { inherit system; modules = [ - ./modules - ./users + ./nixos/modules + ./nixos/users sops-nix.nixosModules.sops + ({ config, pkgs, ... }: { + # Inject git revision for our custom GitOps module to query + system.configurationRevision = self.rev or self.dirtyRev or "unknown"; + }) ]; }; hosts = { - # Physical hosts - Niko.modules = [ ./hosts/Niko ]; - - # Virtual machines - - # Single-service - Ingress.modules = [ ./hosts/Ingress ]; - Gitea.modules = [ ./hosts/Gitea ]; - Vaultwarden.modules = [ ./hosts/Vaultwarden ]; - - # Production multi-service - Binnenpost.modules = [ ./hosts/Binnenpost ]; - Production.modules = [ ./hosts/Production ]; - ProductionGPU.modules = [ ./hosts/ProductionGPU ]; - ProductionArr.modules = [ ./hosts/ProductionArr ]; - ACE.modules = [ ./hosts/ACE ]; - - # Others - Template.modules = [ ./hosts/Template ]; - Development.modules = [ ./hosts/Development ]; - Testing.modules = [ ./hosts/Testing ]; + izanagi.modules = [ ./nixos/hosts/izanagi ]; + Gitea.modules = [ ./nixos/hosts/Gitea ]; }; }; } diff --git a/hosts/ACE/default.nix b/nixos/hosts/ACE/default.nix similarity index 100% rename from hosts/ACE/default.nix rename to nixos/hosts/ACE/default.nix diff --git a/hosts/Binnenpost/default.nix b/nixos/hosts/Binnenpost/default.nix similarity index 100% rename from hosts/Binnenpost/default.nix rename to nixos/hosts/Binnenpost/default.nix diff --git a/nixos/hosts/Builder/default.nix b/nixos/hosts/Builder/default.nix new file mode 100644 index 0000000..515bcb9 --- /dev/null +++ b/nixos/hosts/Builder/default.nix @@ -0,0 +1,50 @@ +{ pkgs, ... }: + +{ + config = { + networking = { + hostName = "Builder"; + hostId = "aaaa4200"; + domain = "depeuter.dev"; + + useDHCP = false; + enableIPv6 = true; + + defaultGateway = { + address = "192.168.0.1"; + interface = "ens18"; + }; + + interfaces.ens18 = { + ipv4.addresses = [ + { + address = "192.168.0.42"; + prefixLength = 24; + } + ]; + }; + + nameservers = [ + "1.1.1.1" + "1.0.0.1" + ]; + }; + + # Dedicated builder user + users.users.builder = { + isNormalUser = true; + description = "Nix Remote Builder"; + # You must add the public key corresponding to `builder-ssh-key` here + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1... TODO: replace with build-farm-key.pub" + ]; + }; + + # Optimize nix settings for a build farm + nix.settings.trusted-users = [ "builder" ]; + nix.settings.cores = 0; # Use all cores + nix.settings.max-jobs = "auto"; + + system.stateVersion = "24.05"; + }; +} diff --git a/hosts/Development/default.nix b/nixos/hosts/Development/default.nix similarity index 100% rename from hosts/Development/default.nix rename to nixos/hosts/Development/default.nix diff --git a/hosts/Gitea/default.nix b/nixos/hosts/Gitea/default.nix similarity index 100% rename from hosts/Gitea/default.nix rename to nixos/hosts/Gitea/default.nix diff --git a/hosts/Ingress/default.nix b/nixos/hosts/Ingress/default.nix similarity index 100% rename from hosts/Ingress/default.nix rename to nixos/hosts/Ingress/default.nix diff --git a/hosts/Isabel/.keep b/nixos/hosts/Isabel/.keep similarity index 100% rename from hosts/Isabel/.keep rename to nixos/hosts/Isabel/.keep diff --git a/hosts/Isabel/dashboard/config/bookmarks.yaml b/nixos/hosts/Isabel/dashboard/config/bookmarks.yaml similarity index 100% rename from hosts/Isabel/dashboard/config/bookmarks.yaml rename to nixos/hosts/Isabel/dashboard/config/bookmarks.yaml diff --git a/hosts/Isabel/dashboard/config/services.yaml b/nixos/hosts/Isabel/dashboard/config/services.yaml similarity index 100% rename from hosts/Isabel/dashboard/config/services.yaml rename to nixos/hosts/Isabel/dashboard/config/services.yaml diff --git a/hosts/Isabel/default.nix b/nixos/hosts/Isabel/default.nix similarity index 100% rename from hosts/Isabel/default.nix rename to nixos/hosts/Isabel/default.nix diff --git a/hosts/Isabel/hardware-configuration.nix b/nixos/hosts/Isabel/hardware-configuration.nix similarity index 100% rename from hosts/Isabel/hardware-configuration.nix rename to nixos/hosts/Isabel/hardware-configuration.nix diff --git a/hosts/Niko/default.nix b/nixos/hosts/Niko/default.nix similarity index 100% rename from hosts/Niko/default.nix rename to nixos/hosts/Niko/default.nix diff --git a/hosts/Niko/hardware-configuration.nix b/nixos/hosts/Niko/hardware-configuration.nix similarity index 100% rename from hosts/Niko/hardware-configuration.nix rename to nixos/hosts/Niko/hardware-configuration.nix diff --git a/hosts/Production/default.nix b/nixos/hosts/Production/default.nix similarity index 100% rename from hosts/Production/default.nix rename to nixos/hosts/Production/default.nix diff --git a/hosts/ProductionArr/default.nix b/nixos/hosts/ProductionArr/default.nix similarity index 100% rename from hosts/ProductionArr/default.nix rename to nixos/hosts/ProductionArr/default.nix diff --git a/hosts/ProductionGPU/default.nix b/nixos/hosts/ProductionGPU/default.nix similarity index 100% rename from hosts/ProductionGPU/default.nix rename to nixos/hosts/ProductionGPU/default.nix diff --git a/hosts/Template/default.nix b/nixos/hosts/Template/default.nix similarity index 100% rename from hosts/Template/default.nix rename to nixos/hosts/Template/default.nix diff --git a/hosts/Testing/default.nix b/nixos/hosts/Testing/default.nix similarity index 100% rename from hosts/Testing/default.nix rename to nixos/hosts/Testing/default.nix diff --git a/hosts/Vaultwarden/default.nix b/nixos/hosts/Vaultwarden/default.nix similarity index 100% rename from hosts/Vaultwarden/default.nix rename to nixos/hosts/Vaultwarden/default.nix diff --git a/nixos/hosts/izanagi/default.nix b/nixos/hosts/izanagi/default.nix new file mode 100644 index 0000000..1d7e09c --- /dev/null +++ b/nixos/hosts/izanagi/default.nix @@ -0,0 +1,52 @@ +{ config, pkgs, ... }: + +{ + networking.hostName = "izanagi"; + + homelab = { + # Enable the standard guest VM configuration + virtualisation.guest.enable = true; + + # Enable standard GitOps for the Control Center itself + gitops.enable = true; + # We might pull from our Forgejo instance eventually, but for bootstrap + # it might need to pull from Github or the local Gitea if it's up. + # gitops.repoUrl = "https://git.depeuter.dev/Bos55/nix-config.git"; + + services = { + openssh.enable = true; + + # Enable the Hypervisor GitOps service to provision OTHER VMs + hypervisor-gitops = { + enable = true; + repoUrl = "https://git.depeuter.dev/Bos55/nix-config.git"; + pollInterval = "hourly"; + }; + }; + }; + + sops = { + defaultSopsFile = ./secrets.yaml; + defaultSopsFormat = "yaml"; + age.keyFile = "/var/lib/sops-age/keys.txt"; # Injected via Cloud-Init during Phase 4 + + secrets = { + proxmox_api_token.owner = "root"; + forgejo_token.owner = "root"; + }; + + templates."hypervisor-gitops.env".content = '' + PROXMOX_VE_API_TOKEN=${config.sops.placeholder.proxmox_api_token} + FORGEJO_TOKEN=${config.sops.placeholder.forgejo_token} + PROXMOX_VE_ENDPOINT=https://mikoshi:8006/ + PROXMOX_VE_INSECURE=true + ''; + }; + + # Make the secrets available to the hypervisor-gitops service via EnvironmentFile + systemd.services.hypervisor-gitops.serviceConfig.EnvironmentFile = [ + config.sops.templates."hypervisor-gitops.env".path + ]; + + system.stateVersion = "24.05"; +} diff --git a/nixos/hosts/izanagi/secrets.yaml b/nixos/hosts/izanagi/secrets.yaml new file mode 100644 index 0000000..76b9924 --- /dev/null +++ b/nixos/hosts/izanagi/secrets.yaml @@ -0,0 +1,7 @@ +# This is a placeholder SOPS file. +# You must encrypt it with sops using your age key before deploying. +# sops -e -i secrets.yaml +proxmox_api_token: "PLACEHOLDER_TOKEN" +forgejo_token: "PLACEHOLDER_TOKEN" +builder-ssh-key: "PLACEHOLDER_KEY" +webhook-secret: "PLACEHOLDER_SECRET" diff --git a/modules/apps/arr/default.nix b/nixos/modules/apps/arr/default.nix similarity index 69% rename from modules/apps/arr/default.nix rename to nixos/modules/apps/arr/default.nix index 7b530c3..8fcc56d 100644 --- a/modules/apps/arr/default.nix +++ b/nixos/modules/apps/arr/default.nix @@ -61,34 +61,30 @@ in { virtualisation.containers.enable = lib.mkIf inUse true; }; - fileSystems = let - mkFileSystem = device: { - inherit device; - fsType = "nfs"; - options = [ - "rw" - "auto" - "nfsvers=4.2" - "rsize=1048576" "wsize=1048576" - "hard" - "timeo=600" "retrans=2" - "_netdev" "nosuid" "tcp" - ]; - }; - + homelab.nfsMounts = let hugoBackup = "192.168.0.11:/mnt/BIG/BACKUP"; + arrOptions = [ + "auto" + "rsize=1048576" "wsize=1048576" + "hard" + "timeo=600" "retrans=2" + "_netdev" + ]; + mkMount = device: { + inherit device; + extraOptions = arrOptions; + }; in lib.mkIf inUse { - "/srv/bazarr-backup" = lib.mkIf cfg.bazarr.enable (mkFileSystem "${hugoBackup}/BAZARR"); - "/srv/prowlarr-backup" = lib.mkIf cfg.bazarr.enable (mkFileSystem "${hugoBackup}/PROWLARR"); - "/srv/qbittorrent" = lib.mkIf cfg.qbittorrent.enable (mkFileSystem "192.168.0.11:/mnt/SMALL/CONFIG/QBITTORRENT"); - "/srv/radarr-backup" = lib.mkIf cfg.radarr.enable (mkFileSystem "${hugoBackup}/RADARR"); - "/srv/sonarr-backup" = lib.mkIf cfg.sonarr.enable (mkFileSystem "${hugoBackup}/SONARR"); - "/srv/torrent" = mkFileSystem "192.168.0.11:/mnt/SMALL/MEDIA/TORRENT"; + "/srv/bazarr-backup" = lib.mkIf cfg.bazarr.enable (mkMount "${hugoBackup}/BAZARR"); + "/srv/prowlarr-backup" = lib.mkIf cfg.bazarr.enable (mkMount "${hugoBackup}/PROWLARR"); + "/srv/qbittorrent" = lib.mkIf cfg.qbittorrent.enable (mkMount "192.168.0.11:/mnt/SMALL/CONFIG/QBITTORRENT"); + "/srv/radarr-backup" = lib.mkIf cfg.radarr.enable (mkMount "${hugoBackup}/RADARR"); + "/srv/sonarr-backup" = lib.mkIf cfg.sonarr.enable (mkMount "${hugoBackup}/SONARR"); + "/srv/torrent" = mkMount "192.168.0.11:/mnt/SMALL/MEDIA/TORRENT"; }; # Make sure the Docker network exists. - systemd.services."docker-${networkName}-create-network" = lib.mkIf inUse { - description = "Create Docker network for ${networkName}"; + homelab.dockerNetworks."${networkName}" = lib.mkIf inUse { requiredBy = [ "docker-bazarr.service" "docker-prowlarr.service" @@ -96,38 +92,47 @@ in { "docker-radarr.service" "docker-sonarr.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then - ${pkgs.docker}/bin/docker network create ${networkName} - fi - ''; }; # Create a user for each app. - users.users = let + homelab.appUsers = let mkUser = uid: { - uid = lib.mkForce uid; - isSystemUser = true; + inherit uid; group = config.users.groups.media.name; - home = "/var/empty"; - shell = null; }; in { bazarr = lib.mkIf cfg.bazarr.enable (mkUser 3003); prowlarr = lib.mkIf cfg.prowlarr.enable (mkUser 3004); - qbittorrent = lib.mkIf cfg.qbittorrent.enable (mkUser 3005) // { - extraGroups = [ - config.users.groups.apps.name - ]; - }; + qbittorrent = lib.mkIf cfg.qbittorrent.enable ((mkUser 3005) // { + extraGroups = [ config.users.groups.apps.name ]; + }); radarr = lib.mkIf cfg.radarr.enable (mkUser 3006); sonarr = lib.mkIf cfg.sonarr.enable (mkUser 3007); }; + homelab.traefikRouters = { + bazarr = lib.mkIf cfg.bazarr.enable { + rule = "Host(`bazarr.depeuter.dev`)"; + port = 6767; + }; + prowlarr = lib.mkIf cfg.prowlarr.enable { + rule = "Host(`prowlarr.depeuter.dev`)"; + port = 9696; + }; + qbittorrent = lib.mkIf cfg.qbittorrent.enable { + rule = "Host(`qb.depeuter.dev`)"; + port = 10095; + }; + radarr = lib.mkIf cfg.radarr.enable { + rule = "Host(`radarr.depeuter.dev`)"; + port = 7878; + }; + sonarr = lib.mkIf cfg.sonarr.enable { + rule = "Host(`sonarr.depeuter.dev`)"; + port = 8989; + }; + }; + virtualisation.oci-containers.containers = let videoHostPath = config.homelab.fileSystems.media.video.hostPath; in { @@ -159,12 +164,6 @@ in { "${videoHostPath}/Films:/media/movies" "${videoHostPath}/Series:/media/series" ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.bazarr.rule" = "Host(`bazarr.depeuter.dev`)"; - "traefik.http.services.bazarr.loadbalancer.server.port" = toString port; - }; }; prowlarr = let @@ -190,12 +189,6 @@ in { "/srv/prowlarr-backup:/config/Backups" ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.prowlarr.rule" = "Host(`prowlarr.depeuter.dev`)"; - "traefik.http.services.prowlarr.loadbalancer.server.port" = toString port; - }; }; qbittorrent = let @@ -223,12 +216,6 @@ in { "/srv/torrent:/media/cache" ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.qbittorrent.rule" = "Host(`qb.depeuter.dev`)"; - "traefik.http.services.qbittorrent.loadbalancer.server.port" = toString port; - }; }; radarr = let @@ -257,12 +244,6 @@ in { "/srv/torrent:/media/cache" "${videoHostPath}/Films:/media/movies" ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.radarr.rule" = "Host(`radarr.depeuter.dev`)"; - "traefik.http.services.radarr.loadbalancer.server.port" = toString port; - }; }; sonarr = let @@ -291,12 +272,6 @@ in { "/srv/torrent:/media/cache" "${videoHostPath}/Series:/media/series" ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.sonarr.rule" = "Host(`sonarr.depeuter.dev`)"; - "traefik.http.services.sonarr.loadbalancer.server.port" = toString port; - }; }; }; }; diff --git a/modules/apps/bind9/db.depeuter.dev b/nixos/modules/apps/bind9/db.depeuter.dev similarity index 100% rename from modules/apps/bind9/db.depeuter.dev rename to nixos/modules/apps/bind9/db.depeuter.dev diff --git a/modules/apps/bind9/default.nix b/nixos/modules/apps/bind9/default.nix similarity index 100% rename from modules/apps/bind9/default.nix rename to nixos/modules/apps/bind9/default.nix diff --git a/modules/apps/bind9/named.conf b/nixos/modules/apps/bind9/named.conf similarity index 100% rename from modules/apps/bind9/named.conf rename to nixos/modules/apps/bind9/named.conf diff --git a/modules/apps/bind9/named.conf.local b/nixos/modules/apps/bind9/named.conf.local similarity index 100% rename from modules/apps/bind9/named.conf.local rename to nixos/modules/apps/bind9/named.conf.local diff --git a/modules/apps/bind9/named.conf.options b/nixos/modules/apps/bind9/named.conf.options similarity index 100% rename from modules/apps/bind9/named.conf.options rename to nixos/modules/apps/bind9/named.conf.options diff --git a/modules/apps/calibre/default.nix b/nixos/modules/apps/calibre/default.nix similarity index 69% rename from modules/apps/calibre/default.nix rename to nixos/modules/apps/calibre/default.nix index bddf5c8..e7847da 100644 --- a/modules/apps/calibre/default.nix +++ b/nixos/modules/apps/calibre/default.nix @@ -34,58 +34,48 @@ in { virtualisation.containers.enable = true; }; - users.users.calibre = { - uid = lib.mkForce 3010; - isSystemUser = true; + homelab.appUsers.calibre = { + uid = 3010; group = config.users.groups.media.name; - home = "/var/empty"; - shell = null; }; - fileSystems."${books}" = { + homelab.nfsMounts."${books}" = { device = "192.168.0.11:/mnt/SMALL/MEDIA/BOOKS"; - fsType = "nfs"; - options = [ - "rw" + extraOptions = [ "auto" - "nfsvers=4.2" "rsize=1048576" "wsize=1048576" - "soft" "timeo=600" "retrans=2" - "_netdev" "nosuid" "tcp" + "_netdev" ]; }; # Make sure the Docker network exists. - systemd.services."docker-${networkName}-create-network" = { + homelab.dockerNetworks."${networkName}" = { requiredBy = [ "docker-calibre.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; + }; + homelab.traefikRouters = { + calibre = lib.mkIf cfg.desktop.enable { + rule = "Host(`calibre.depeuter.dev`)"; + port = 8080; + }; + calibre-web = lib.mkIf cfg.web.enable { + rule = "Host(`books.depeuter.dev`)"; + port = 8083; }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then - ${pkgs.docker}/bin/docker network create ${networkName} - fi - ''; }; }) # Calibre desktop (lib.mkIf cfg.desktop.enable { - fileSystems."${calibre-config}" = { + homelab.nfsMounts."${calibre-config}" = { device = "192.168.0.11:/mnt/SMALL/CONFIG/CALIBRE"; - fsType = "nfs"; - options = [ - "rw" + extraOptions = [ "auto" - "nfsvers=4.2" "rsize=1048576" "wsize=1048576" - "soft" "timeo=600" "retrans=2" - "_netdev" "nosuid" "tcp" + "_netdev" ]; }; @@ -122,28 +112,18 @@ in { "${books}:/media/books" ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.calibre.rule" = "Host(`calibre.depeuter.dev`)"; - "traefik.http.services.calibre.loadbalancer.server.port" = toString innerPort; - }; }; }) # Calibre Web (lib.mkIf cfg.web.enable { - fileSystems."${calibre-web-config}" = { + homelab.nfsMounts."${calibre-web-config}" = { device = "192.168.0.11:/mnt/SMALL/CONFIG/CALIBRE-WEB"; - fsType = "nfs"; - options = [ - "rw" + extraOptions = [ "auto" - "nfsvers=4.2" "rsize=1048576" "wsize=1048576" - "soft" "timeo=600" "retrans=2" - "_netdev" "nosuid" "tcp" + "_netdev" ]; }; @@ -177,12 +157,6 @@ in { "${books}:/media/books" ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.calibre-web.rule" = "Host(`books.depeuter.dev`)"; - "traefik.http.services.calibre-web.loadbalancer.server.port" = toString innerPort; - }; }; }) ]; diff --git a/modules/apps/changedetection/default.nix b/nixos/modules/apps/changedetection/default.nix similarity index 100% rename from modules/apps/changedetection/default.nix rename to nixos/modules/apps/changedetection/default.nix diff --git a/modules/apps/default.nix b/nixos/modules/apps/default.nix similarity index 93% rename from modules/apps/default.nix rename to nixos/modules/apps/default.nix index 385f915..88f35d5 100644 --- a/modules/apps/default.nix +++ b/nixos/modules/apps/default.nix @@ -8,6 +8,7 @@ ./gitea ./homepage ./jellyfin + ./monitoring ./plex ./solidtime ./speedtest diff --git a/modules/apps/freshrss/default.nix b/nixos/modules/apps/freshrss/default.nix similarity index 65% rename from modules/apps/freshrss/default.nix rename to nixos/modules/apps/freshrss/default.nix index 4f4456f..e997fc5 100644 --- a/modules/apps/freshrss/default.nix +++ b/nixos/modules/apps/freshrss/default.nix @@ -4,6 +4,7 @@ let cfg = config.homelab.apps.freshrss; networkName = "freshrss"; + proxyNet = config.homelab.apps.traefik.sharedNetworkName; in { options.homelab.apps.freshrss = { enable = lib.mkEnableOption "FreshRSS"; @@ -20,35 +21,36 @@ in { lib.mkIf cfg.enable { homelab.virtualisation.containers.enable = true; - fileSystems."/srv/freshrss" = { + homelab.nfsMounts."/srv/freshrss" = { device = "192.168.0.11:/mnt/SMALL/CONFIG/FRESHRSS"; - fsType = "nfs"; - options = [ - "rw" + extraOptions = [ "auto" - "nfsvers=4.2" - "async" "soft" "timeo=600" + "timeo=600" "retrans=2" "_netdev" - "nosuid" - "tcp" ]; }; - systemd.services."docker-${networkName}-create-network" = { - description = "Create Docker network for ${networkName}"; + homelab.dockerNetworks."${networkName}" = { requiredBy = [ "docker-freshrss.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; + }; + + homelab.traefikRouters.freshrss = { + rule = "Host(`rss.depeuter.dev`)"; + port = 80; + tls = true; + entryPoints = [ "websecure" ]; + middlewares = [ "freshrssM1" "freshrssM2" ]; + extraLabels = { + "traefik.http.middlewares.freshrssM1.compress" = "true"; + "traefik.http.middlewares.freshrssM2.headers.browserXssFilter" = "true"; + "traefik.http.middlewares.freshrssM2.headers.forceSTSHeader" = "true"; + "traefik.http.middlewares.freshrssM2.headers.frameDeny" = "true"; + "traefik.http.middlewares.freshrssM2.headers.referrerPolicy" = "no-referrer-when-downgrade"; + "traefik.http.middlewares.freshrssM2.headers.stsSeconds" = "31536000"; }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then - ${pkgs.docker}/bin/docker network create ${networkName} - fi - ''; }; virtualisation.oci-containers.containers.freshrss = { @@ -61,6 +63,7 @@ in { ]; extraOptions = [ "--network=${networkName}" + "--network=${proxyNet}" ]; environment = { TZ = config.time.timeZone; @@ -72,22 +75,6 @@ in { "/srv/freshrss/www/freshrss/data:/var/www/FreshRSS/data" "/srv/freshrss/www/freshrss/extensions:/var/www/FreshRSS/extensions" ]; - labels = { - "traefik.enable" = "true"; - - "traefik.http.middlewares.freshrssM1.compress" = "true"; - "traefik.http.middlewares.freshrssM2.headers.browserXssFilter" = "true"; - "traefik.http.middlewares.freshrssM2.headers.forceSTSHeader" = "true"; - "traefik.http.middlewares.freshrssM2.headers.frameDeny" = "true"; - "traefik.http.middlewares.freshrssM2.headers.referrerPolicy" = "no-referrer-when-downgrade"; - "traefik.http.middlewares.freshrssM2.headers.stsSeconds" = "31536000"; - "traefik.http.routers.freshrss.entryPoints" = "websecure"; - "traefik.http.routers.freshrss.tls" = "true"; - - "traefik.http.services.freshrss.loadbalancer.server.port" = "80"; - "traefik.http.routers.freshrss.middlewares" = "freshrssM1,freshrssM2"; - "traefik.http.routers.freshrss.rule" = "Host(`rss.depeuter.dev`)"; - }; }; }; } diff --git a/modules/apps/gitea/default.nix b/nixos/modules/apps/gitea/default.nix similarity index 95% rename from modules/apps/gitea/default.nix rename to nixos/modules/apps/gitea/default.nix index 0361bd5..0b90c3d 100644 --- a/modules/apps/gitea/default.nix +++ b/nixos/modules/apps/gitea/default.nix @@ -21,6 +21,51 @@ in { options.homelab.apps.gitea.enable = lib.mkEnableOption "Gitea"; config = lib.mkIf cfg.enable { + services.nats = { + enable = true; + port = 4222; + jetstream = { + enable = true; + }; + }; + + networking.firewall.allowedTCPPorts = [ 4222 9000 ]; + + systemd.services.gitea-webhook-bridge = { + description = "Gitea Webhook Bridge to NATS JetStream"; + after = [ "network.target" "nats.service" ]; + wants = [ "nats.service" ]; + wantedBy = [ "multi-user.target" ]; + path = with pkgs; [ natscli python3 ]; + script = '' + python3 -c ' +import http.server +import subprocess +import sys + +class WebhookHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + content_length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(content_length) + try: + subprocess.run(["nats", "pub", "--server=nats://127.0.0.1:4222", "forgejo.staging"], input=body, check=True) + self.send_response(200) + self.end_headers() + self.wfile.write(b"OK\n") + except Exception as e: + self.send_response(500) + self.end_headers() + self.wfile.write(str(e).encode("utf-8")) + + def log_message(self, format, *args): + sys.stderr.write("%s - - [%s] %s\n" % (self.client_address[0], self.log_date_time_string(), format%args)) + +server = http.server.ThreadingHTTPServer(("0.0.0.0", 9000), WebhookHandler) +server.serve_forever() +' + ''; + }; + homelab = { users = { apps.enable = true; @@ -30,61 +75,28 @@ in { virtualisation.containers.enable = true; }; - users.users.gitea = { - uid = lib.mkForce UID; - isSystemUser = true; - group = config.users.groups.apps.name; - home = "/var/empty"; - shell = null; + homelab.appUsers.gitea = { + uid = UID; }; # Use filesystem mounts because rootless containers otherwise don't have access to the mount path (nested in docker directories). # You could probably fix this by modifying the access rights on the path, but what would the point of that be? - fileSystems = { - # Mount options: - # - hard: retry requests indefinitely if the server becomes unresponsive. - # - nosuid: prevent set-user-id and set-group-id bits + homelab.nfsMounts = { "/srv/gitea-config" = { device = "192.168.0.11:/mnt/SMALL/CONFIG/GITEA"; - fsType = "nfs"; - options = [ - "rw" - "nfsvers=4.2" - "async" "soft" "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all" - "nosuid" - "tcp" - ]; }; "/srv/gitea-git" = { device = "192.168.0.11:/mnt/SMALL/DATA/GIT"; - fsType = "nfs"; - options = [ - "rw" - "nfsvers=4.2" - "async" "soft" "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all" - "nosuid" - "tcp" - ]; }; }; # Make sure the Docker network exists. - systemd.services."docker-${networkName}-create-network" = { - description = "Create Docker network for ${networkName}"; + homelab.dockerNetworks."${networkName}" = { requiredBy = [ "docker-gitea-db.service" "docker-gitea.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then - ${pkgs.docker}/bin/docker network create ${networkName} - fi - ''; }; virtualisation.oci-containers.containers = { @@ -471,6 +483,7 @@ in { # ... oath2_client # ... webhook + FORGEJO__webhook__ALLOWED_HOST_LIST = "192.168.0.0/16,127.0.0.0/8,host.docker.internal,*"; FORGEJO__mailer__ENABLED = "true"; # Buffer length of channel, keep it as it is if you don't know what it is. diff --git a/modules/apps/homepage/default.nix b/nixos/modules/apps/homepage/default.nix similarity index 69% rename from modules/apps/homepage/default.nix rename to nixos/modules/apps/homepage/default.nix index b34f32f..b820e2b 100644 --- a/modules/apps/homepage/default.nix +++ b/nixos/modules/apps/homepage/default.nix @@ -26,26 +26,24 @@ in { virtualisation.containers.enable = true; }; - users.users.homepage = { - uid = lib.mkForce 3018; - isSystemUser = true; - group = config.users.groups.apps.name; - home = "/var/empty"; - shell = null; + homelab.appUsers.homepage = { + uid = 3018; }; - fileSystems."${homepage-config}" = { + homelab.nfsMounts."${homepage-config}" = { device = "192.168.0.11:/mnt/SMALL/CONFIG/HOMEPAGE"; - fsType = "nfs"; - options = [ - "rw" + extraOptions = [ "auto" - "nfsvers=4.2" - "async" "soft" "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all" - "nosuid" "tcp" ]; }; + homelab.traefikRouters.homepage = let + host = "homepage.${config.networking.domain}"; + in { + rule = "Host(`${host}`)"; + port = cfg.port; + }; + virtualisation.oci-containers.containers.homepage = let host = "homepage.${config.networking.domain}"; in { @@ -63,12 +61,6 @@ in { "${homepage-config}:/app/config" # "/var/run/docker.sock:/var/run/docker.sock:ro" # For docker integrations ]; - labels = { - "traefik.enable" = "true"; - "traefik.docker.network" = proxyNet; - "traefik.http.routers.homepage.rule" = "Host(`${host}`)"; - "traefik.http.services.homepage.loadbalancer.server.port" = toString cfg.port; - }; environment = { inherit PUID PGID; diff --git a/modules/apps/jellyfin/default.nix b/nixos/modules/apps/jellyfin/default.nix similarity index 58% rename from modules/apps/jellyfin/default.nix rename to nixos/modules/apps/jellyfin/default.nix index 011f56b..09bbbb5 100644 --- a/modules/apps/jellyfin/default.nix +++ b/nixos/modules/apps/jellyfin/default.nix @@ -4,6 +4,7 @@ let cfg = config.homelab.apps.jellyfin; networkName = "jellyfin"; + proxyNet = config.homelab.apps.traefik.sharedNetworkName; inherit (config.homelab.fileSystems) media; UID = 3008; @@ -25,71 +26,36 @@ in { virtualisation.containers.enable = true; }; - fileSystems = { - "/srv/audio" = { - device = "192.168.0.11:/mnt/SMALL/MEDIA/AUDIO"; - fsType = "nfs"; - options = [ - "ro" - "nfsvers=4.2" - "async" "soft" - "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all" - "nosuid" "tcp" - ]; - }; - - "/srv/homevideo" = { - device = "192.168.0.11:/mnt/BIG/MEDIA/HOMEVIDEO/ARCHIVE"; - fsType = "nfs"; - options = [ - "ro" - "nfsvers=4.2" - "async" "soft" - "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all" - "nosuid" "tcp" - ]; - }; - - "/srv/photo" = { - device = "192.168.0.11:/mnt/BIG/MEDIA/PHOTO/ARCHIVE"; - fsType = "nfs"; - options = [ - "ro" - "nfsvers=4.2" - "async" "soft" - "timeo=100" "retry=50" "actimeo=1800" "lookupcache=all" - "nosuid" "tcp" - ]; + homelab.nfsMounts = let + mkMount = device: { + inherit device; + readOnly = true; }; + in { + "/srv/audio" = mkMount "192.168.0.11:/mnt/SMALL/MEDIA/AUDIO"; + "/srv/homevideo" = mkMount "192.168.0.11:/mnt/BIG/MEDIA/HOMEVIDEO/ARCHIVE"; + "/srv/photo" = mkMount "192.168.0.11:/mnt/BIG/MEDIA/PHOTO/ARCHIVE"; }; - users.users.jellyfin = { - uid = lib.mkForce UID; - isSystemUser = true; - group = config.users.groups.apps.name; - extraGroups = [ - config.users.groups.media.name - ]; - home = "/var/empty"; - shell = null; + homelab.appUsers.jellyfin = { + uid = UID; + extraGroups = [ config.users.groups.media.name ]; }; # Make sure the Docker network exists. - systemd.services."docker-${networkName}-create-network" = { - description = "Create Docker network for ${networkName}"; + homelab.dockerNetworks."${networkName}" = { requiredBy = [ "docker-jellyfin.service" "docker-feishin.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; + }; + + homelab.traefikRouters.feishin = { + rule = "Host(`play.jelly.depeuter.dev`)"; + port = 9180; + extraLabels = { + "traefik.tls.options.default.minVersion" = "VersionTLS13"; }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then - ${pkgs.docker}/bin/docker network create ${networkName} - fi - ''; }; virtualisation.oci-containers.containers = { @@ -145,6 +111,7 @@ in { ]; extraOptions = [ "--network=${networkName}" + "--network=${proxyNet}" ]; environment = { # pre defined server name @@ -157,12 +124,6 @@ in { SERVER_URL= "https://jelly.depeuter.dev"; TZ = config.time.timeZone; }; - labels = { - "traefik.enable" = "true"; - "traefik.http.routers.feishin.rule" = "Host(`play.jelly.depeuter.dev`)"; - "traefik.http.services.feishin.loadbalancer.server.port" = feishinPort; - "traefik.tls.options.default.minVersion" = "VersionTLS13"; - }; }; }; }; diff --git a/nixos/modules/apps/monitoring/default.nix b/nixos/modules/apps/monitoring/default.nix new file mode 100644 index 0000000..3e79b25 --- /dev/null +++ b/nixos/modules/apps/monitoring/default.nix @@ -0,0 +1,164 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.homelab.apps.monitoring; +in { + options.homelab.apps.monitoring.enable = lib.mkEnableOption "Homelab Monitoring Stack"; + + config = lib.mkIf cfg.enable { + sops.secrets."grafana/admin_password" = { + sopsFile = ../../../../secrets/prod/monitoring.yaml; + }; + sops.secrets."alertmanager/smtp_password" = { + sopsFile = ../../../../secrets/prod/monitoring.yaml; + }; + + # 1. Loki Log Storage + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server.http_listen_port = 3100; + common.ring.instance_addr = "127.0.0.1"; + common.ring.kvstore.store = "inmemory"; + schema_config = { + configs = [{ + from = "2020-10-24"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "24h"; + }; + }]; + }; + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + }; + filesystem.directory = "/var/lib/loki/chunks"; + }; + }; + }; + + # 2. Prometheus Time-Series DB + services.prometheus = { + enable = true; + port = 9090; + scrapeConfigs = [ + { + job_name = "node"; + scrape_interval = "15s"; + # In a real setup, we would use Prometheus service discovery (e.g., file_sd_configs) + # or list all homelab IPs here. For now, we scrape localhost. + static_configs = [{ + targets = [ "127.0.0.1:9100" ]; + }]; + } + ]; + # Connect Prometheus to Alertmanager + alertmanagers = [{ + static_configs = [{ + targets = [ "127.0.0.1:9093" ]; + }]; + }]; + }; + + # 3. Alertmanager (Routing alerts to NTFY and Email) + services.prometheus.alertmanager = { + enable = true; + port = 9093; + configuration = { + global = { + smtp_smarthost = "smtp.example.com:587"; + smtp_from = "alerts@depeuter.dev"; + smtp_auth_username = "alerts@depeuter.dev"; + smtp_auth_password_file = config.sops.secrets."alertmanager/smtp_password".path; + }; + route = { + receiver = "ntfy-and-email"; + group_wait = "30s"; + group_interval = "5m"; + repeat_interval = "4h"; + group_by = [ "alertname" "instance" ]; + }; + receivers = [{ + name = "ntfy-and-email"; + email_configs = [{ + to = "your-email@example.com"; + # Use smarthost settings defined in global + }]; + webhook_configs = [{ + # Alertmanager natively supports webhooks. We send the JSON to NTFY's Prometheus endpoint. + url = "http://127.0.0.1:2586/alerts"; + }]; + }]; + }; + }; + + # 4. NTFY Push Notification Server + services.ntfy-sh = { + enable = true; + settings = { + base-url = "https://ntfy.lab.depeuter.dev"; + listen-http = ":2586"; + # You can configure auth via the CLI once the service is running, or via auth-file. + # For now, it is open locally. + }; + }; + + # 5. Grafana Visualization + services.grafana = { + enable = true; + settings.server = { + http_port = 3000; + http_addr = "127.0.0.1"; + domain = "grafana.lab.depeuter.dev"; + }; + settings.security.admin_password = "$__file{${config.sops.secrets."grafana/admin_password".path}}"; + + # Declarative Data Sources + provision = { + enable = true; + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + access = "proxy"; + url = "http://127.0.0.1:9090"; + isDefault = true; + } + { + name = "Loki"; + type = "loki"; + access = "proxy"; + url = "http://127.0.0.1:3100"; + } + ]; + }; + }; + + # 6. Traefik Reverse Proxy for Grafana and NTFY + homelab.apps.traefik.dynamicConfigOptions.http = { + routers = { + grafana = { + rule = "Host(`grafana.lab.depeuter.dev`)"; + service = "grafana"; + }; + ntfy = { + rule = "Host(`ntfy.lab.depeuter.dev`)"; + service = "ntfy"; + }; + }; + services = { + grafana.loadBalancer.servers = [{ url = "http://host.docker.internal:3000"; }]; + ntfy.loadBalancer.servers = [{ url = "http://host.docker.internal:2586"; }]; + }; + }; + + # Open firewall for Loki so agents can push logs + networking.firewall.allowedTCPPorts = [ 3100 ]; + }; +} diff --git a/modules/apps/plex/default.nix b/nixos/modules/apps/plex/default.nix similarity index 100% rename from modules/apps/plex/default.nix rename to nixos/modules/apps/plex/default.nix diff --git a/modules/apps/solidtime/default.nix b/nixos/modules/apps/solidtime/default.nix similarity index 89% rename from modules/apps/solidtime/default.nix rename to nixos/modules/apps/solidtime/default.nix index 725d32d..80202b0 100644 --- a/modules/apps/solidtime/default.nix +++ b/nixos/modules/apps/solidtime/default.nix @@ -92,24 +92,13 @@ in { homelab.virtualisation.containers.enable = true; # Make sure the Docker network exists. - systemd.services = { - "docker-${networkName}-create-network" = { - description = "Create Docker network for ${networkName}"; + homelab.dockerNetworks = { + "${networkName}" = { requiredBy = [ "${containers.solidtime.serviceName}.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then - ${pkgs.docker}/bin/docker network create ${networkName} - fi - ''; }; - "docker-${internalNetworkName}-create-network" = { - description = "Create Docker network for ${internalNetworkName}"; + "${internalNetworkName}" = { requiredBy = [ "${containers.solidtime.serviceName}.service" "${containers.solidtimeScheduler.serviceName}.service" @@ -117,15 +106,6 @@ in { "${containers.solidtimeDb.serviceName}.service" "${containers.solidtimeGotenberg.serviceName}.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${internalNetworkName}; then - ${pkgs.docker}/bin/docker network create ${internalNetworkName} - fi - ''; }; }; diff --git a/modules/apps/speedtest/default.nix b/nixos/modules/apps/speedtest/default.nix similarity index 100% rename from modules/apps/speedtest/default.nix rename to nixos/modules/apps/speedtest/default.nix diff --git a/modules/apps/technitium-dns/default.nix b/nixos/modules/apps/technitium-dns/default.nix similarity index 100% rename from modules/apps/technitium-dns/default.nix rename to nixos/modules/apps/technitium-dns/default.nix diff --git a/modules/apps/traefik/default.nix b/nixos/modules/apps/traefik/default.nix similarity index 78% rename from modules/apps/traefik/default.nix rename to nixos/modules/apps/traefik/default.nix index 7f6ce38..452df43 100644 --- a/modules/apps/traefik/default.nix +++ b/nixos/modules/apps/traefik/default.nix @@ -12,26 +12,21 @@ in { default = "traefik"; description = "The name of the shared network to connect the container to."; }; + dynamicConfigOptions = lib.mkOption { + type = lib.types.attrs; + default = {}; + description = "Dynamic configuration options to write to file and mount into Traefik."; + }; }; config = lib.mkIf cfg.enable { homelab.virtualisation.containers.enable = true; # Make sure the Docker network exists. - systemd.services."docker-${cfg.sharedNetworkName}-create-network" = { - description = "Create Docker network for ${cfg.sharedNetworkName}"; + homelab.dockerNetworks."${cfg.sharedNetworkName}" = { requiredBy = [ "docker-traefik.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${cfg.sharedNetworkName}; then - ${pkgs.docker}/bin/docker network create ${cfg.sharedNetworkName} - fi - ''; }; virtualisation.oci-containers.containers.traefik = { @@ -45,6 +40,7 @@ in { ]; extraOptions = [ "--network=${cfg.sharedNetworkName}" + "--add-host=host.docker.internal:host-gateway" ]; environmentFiles = [ /home/admin/.cloudflare.secret @@ -56,6 +52,10 @@ in { "--providers.docker=true" "--providers.docker.exposedByDefault=false" + # Add File provider + "--providers.file.filename=/etc/traefik/dynamic_conf.yml" + "--providers.file.watch=true" + # Add web entrypoint "--entrypoints.web.address=:80/tcp" "--entrypoints.web.http.redirections.entrypoint.to=websecure" @@ -75,10 +75,14 @@ in { "--certificatesresolvers.letsencrypt.acme.email=tibo.depeuter@telenet.be" "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" ]; - volumes = [ + volumes = let + dynamicConfFormat = pkgs.formats.yaml { }; + dynamicConfFile = dynamicConfFormat.generate "traefik-dynamic-conf.yml" cfg.dynamicConfigOptions; + in [ "letsencryp:/letsencrypt" "/var/run/docker.sock:/var/run/docker.sock:ro" + "${dynamicConfFile}:/etc/traefik/dynamic_conf.yml:ro" ]; labels = { "traefik.enable" = "true"; diff --git a/modules/apps/vaultwarden/default.nix b/nixos/modules/apps/vaultwarden/default.nix similarity index 98% rename from modules/apps/vaultwarden/default.nix rename to nixos/modules/apps/vaultwarden/default.nix index 4510299..5e9b891 100644 --- a/modules/apps/vaultwarden/default.nix +++ b/nixos/modules/apps/vaultwarden/default.nix @@ -33,21 +33,11 @@ in { }; # Make sure the Docker network exists. - systemd.services."docker-${networkName}-create-network" = { - description = "Create Docker network for ${networkName}"; + homelab.dockerNetworks."${networkName}" = { requiredBy = [ "docker-vaultwarden-db.service" "docker-vaultwarden.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then - ${pkgs.docker}/bin/docker network create ${networkName} - fi - ''; }; virtualisation.oci-containers.containers = let diff --git a/modules/common/default.nix b/nixos/modules/common/default.nix similarity index 63% rename from modules/common/default.nix rename to nixos/modules/common/default.nix index 44309f5..eab0450 100644 --- a/modules/common/default.nix +++ b/nixos/modules/common/default.nix @@ -1,8 +1,18 @@ { + imports = [ + ./docker.nix + ./gitops.nix + ./monitoring.nix + ./nfs.nix + ./traefik.nix + ./users.nix + ]; + config = { homelab = { services.openssh.enable = true; users.admin.enable = true; + gitops.enable = true; }; nix.settings.experimental-features = [ diff --git a/nixos/modules/common/docker.nix b/nixos/modules/common/docker.nix new file mode 100644 index 0000000..293da32 --- /dev/null +++ b/nixos/modules/common/docker.nix @@ -0,0 +1,38 @@ +{ config, lib, pkgs, ... }: + +{ + options.homelab.dockerNetworks = lib.mkOption { + description = "Declarative Docker networks to create before containers start."; + default = {}; + type = lib.types.attrsOf (lib.types.submodule { + options = { + requiredBy = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "List of systemd services that require this network (e.g., docker-containerName.service)."; + }; + }; + }); + }; + + config = { + systemd.services = lib.mapAttrs' (networkName: cfg: + lib.nameValuePair "docker-${networkName}-create-network" { + description = "Create Docker network for ${networkName}"; + requiredBy = cfg.requiredBy; + after = [ "network.target" "docker.service" ]; + requires = [ "docker.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = pkgs.writeShellScript "create-${networkName}-docker-network" '' + if ! ${pkgs.docker}/bin/docker network ls | grep -q ${networkName}; then + ${pkgs.docker}/bin/docker network create ${networkName} + fi + ''; + }; + } + ) config.homelab.dockerNetworks; + }; +} diff --git a/nixos/modules/common/gitops.nix b/nixos/modules/common/gitops.nix new file mode 100644 index 0000000..4bfaeee --- /dev/null +++ b/nixos/modules/common/gitops.nix @@ -0,0 +1,103 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.homelab.gitops; + + updateScript = pkgs.writeShellApplication { + name = "homelab-gitops-update"; + runtimeInputs = [ pkgs.git pkgs.nixos-rebuild pkgs.jq pkgs.coreutils ]; + text = builtins.readFile ../../../scripts/nixos-sync.sh; + }; + +in { + options.homelab.gitops = { + enable = lib.mkEnableOption "Custom GitOps native deployment system"; + + repoUrl = lib.mkOption { + type = lib.types.str; + default = "https://git.depeuter.dev/Bos55/nix-config.git"; + description = "The repository URL to pull configurations from."; + }; + + branch = lib.mkOption { + type = lib.types.str; + default = "v2"; + description = "The branch to deploy."; + }; + + useBuilder = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether to use the central Builder host to compile packages."; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.homelab-gitops = { + description = "Homelab GitOps Update Service"; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${updateScript}/bin/homelab-gitops-update ${cfg.repoUrl} ${cfg.branch}"; + # Must run as root to rebuild the system + User = "root"; + }; + }; + + systemd.timers.homelab-gitops = { + description = "Timer for Homelab GitOps Update Service"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "5m"; + OnUnitActiveSec = "5m"; + RandomizedDelaySec = "30s"; + }; + }; + + sops.secrets."webhook-secret" = {}; + + services.webhook = { + enable = true; + port = 9000; + hooks = { + gitops = { + execute-command = "${pkgs.systemd}/bin/systemctl"; + pass-arguments-to-command = [ + { source = "string"; name = "start"; } + { source = "string"; name = "homelab-gitops.service"; } + ]; + trigger-rule = { + match = { + type = "payload-hash-sha256"; + secret = "{{ getenv \"WEBHOOK_SECRET\" }}"; + parameter = { + source = "header"; + name = "X-Forgejo-Signature"; + }; + }; + }; + }; + }; + }; + + # Inject the secret as an environment variable into the webhook service + systemd.services.webhook.serviceConfig.EnvironmentFile = config.sops.secrets."webhook-secret".path; + + sops.secrets."builder-ssh-key" = lib.mkIf cfg.useBuilder {}; + + nix.buildMachines = lib.mkIf cfg.useBuilder [ + { + hostName = "builder.depeuter.dev"; # Must be routable from nodes + system = "x86_64-linux"; + sshUser = "builder"; + sshKey = config.sops.secrets."builder-ssh-key".path; + maxJobs = 4; + speedFactor = 2; + supportedFeatures = [ "nixos-test" "benchmark" "big-parallel" "kvm" ]; + } + ]; + + nix.distributedBuilds = lib.mkIf cfg.useBuilder true; + }; +} diff --git a/nixos/modules/common/monitoring.nix b/nixos/modules/common/monitoring.nix new file mode 100644 index 0000000..6aeac60 --- /dev/null +++ b/nixos/modules/common/monitoring.nix @@ -0,0 +1,47 @@ +{ config, lib, pkgs, ... }: + +{ + # Prometheus Node Exporter for hardware metrics + services.prometheus.exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9100; + }; + }; + + # Promtail to ship logs to Loki + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 28183; + grpc_listen_port = 0; + }; + positions = { + filename = "/tmp/positions.yaml"; + }; + clients = [{ + # Use the internal DNS name for the Loki ingress + url = "http://loki.lab.depeuter.dev/loki/api/v1/push"; + }]; + scrape_configs = [{ + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + }]; + }; + }; + + # Open firewall ports for node-exporter so Prometheus can scrape it + networking.firewall.allowedTCPPorts = [ 9100 ]; +} diff --git a/nixos/modules/common/nfs.nix b/nixos/modules/common/nfs.nix new file mode 100644 index 0000000..251676f --- /dev/null +++ b/nixos/modules/common/nfs.nix @@ -0,0 +1,46 @@ +{ config, lib, ... }: + +{ + options.homelab.nfsMounts = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + device = lib.mkOption { + type = lib.types.str; + description = "The NFS device, e.g. 192.168.0.11:/mnt/POOL/DATA"; + }; + readOnly = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Whether to mount the NFS share read-only"; + }; + extraOptions = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "async" + "soft" + "timeo=100" + "retry=50" + "actimeo=1800" + "lookupcache=all" + ]; + description = "Extra NFS mount options to append"; + }; + }; + }); + default = {}; + description = "NFS mounts to automatically configure with standard homelab options"; + }; + + config = { + fileSystems = lib.mapAttrs (path: cfg: { + device = cfg.device; + fsType = "nfs"; + options = [ + "nfsvers=4.2" + "nosuid" + "tcp" + ] ++ (if cfg.readOnly then [ "ro" ] else [ "rw" ]) + ++ cfg.extraOptions; + }) config.homelab.nfsMounts; + }; +} diff --git a/nixos/modules/common/traefik.nix b/nixos/modules/common/traefik.nix new file mode 100644 index 0000000..81d1c10 --- /dev/null +++ b/nixos/modules/common/traefik.nix @@ -0,0 +1,65 @@ +{ config, lib, ... }: + +let + proxyNet = config.homelab.apps.traefik.sharedNetworkName; +in { + options.homelab.traefikRouters = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + rule = lib.mkOption { + type = lib.types.str; + description = "The Traefik router rule, e.g. Host(`example.com`)"; + }; + port = lib.mkOption { + type = lib.types.int; + description = "The port the service listens on"; + }; + middlewares = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Middlewares to apply"; + }; + tls = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Whether to enable TLS"; + }; + entryPoints = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Entrypoints to use"; + }; + extraLabels = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = {}; + description = "Extra labels to apply"; + }; + }; + }); + default = {}; + description = "Declarative Traefik router configuration"; + }; + + config = { + # Generate labels for containers based on homelab.traefikRouters + # This assumes that the name in homelab.traefikRouters matches the container name. + virtualisation.oci-containers.containers = lib.mapAttrs (name: router: { + labels = { + "traefik.enable" = "true"; + "traefik.docker.network" = proxyNet; + "traefik.http.routers.${name}.rule" = router.rule; + "traefik.http.services.${name}.loadbalancer.server.port" = toString router.port; + } + // lib.optionalAttrs (router.middlewares != []) { + "traefik.http.routers.${name}.middlewares" = builtins.concatStringsSep "," router.middlewares; + } + // lib.optionalAttrs router.tls { + "traefik.http.routers.${name}.tls" = "true"; + } + // lib.optionalAttrs (router.entryPoints != []) { + "traefik.http.routers.${name}.entryPoints" = builtins.concatStringsSep "," router.entryPoints; + } + // router.extraLabels; + }) config.homelab.traefikRouters; + }; +} diff --git a/nixos/modules/common/users.nix b/nixos/modules/common/users.nix new file mode 100644 index 0000000..59aaea0 --- /dev/null +++ b/nixos/modules/common/users.nix @@ -0,0 +1,37 @@ +{ config, lib, ... }: + +{ + options.homelab.appUsers = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + uid = lib.mkOption { + type = lib.types.int; + description = "The user ID for the app user"; + }; + group = lib.mkOption { + type = lib.types.str; + default = config.users.groups.apps.name; + description = "The primary group for the app user"; + }; + extraGroups = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Extra groups for the app user"; + }; + }; + }); + default = {}; + description = "App users to automatically create with standard homelab options"; + }; + + config = { + users.users = lib.mapAttrs (name: cfg: { + uid = lib.mkForce cfg.uid; + isSystemUser = true; + group = cfg.group; + extraGroups = cfg.extraGroups; + home = "/var/empty"; + shell = null; + }) config.homelab.appUsers; + }; +} diff --git a/modules/default.nix b/nixos/modules/default.nix similarity index 99% rename from modules/default.nix rename to nixos/modules/default.nix index 1a000c3..08bcacd 100644 --- a/modules/default.nix +++ b/nixos/modules/default.nix @@ -1,10 +1,9 @@ { imports = [ ./apps + ./common ./fileSystems ./services ./virtualisation - - ./common ]; } diff --git a/modules/fileSystems/default.nix b/nixos/modules/fileSystems/default.nix similarity index 100% rename from modules/fileSystems/default.nix rename to nixos/modules/fileSystems/default.nix diff --git a/modules/fileSystems/media/default.nix b/nixos/modules/fileSystems/media/default.nix similarity index 100% rename from modules/fileSystems/media/default.nix rename to nixos/modules/fileSystems/media/default.nix diff --git a/modules/fileSystems/media/video/default.nix b/nixos/modules/fileSystems/media/video/default.nix similarity index 76% rename from modules/fileSystems/media/video/default.nix rename to nixos/modules/fileSystems/media/video/default.nix index e46193c..55b6068 100644 --- a/modules/fileSystems/media/video/default.nix +++ b/nixos/modules/fileSystems/media/video/default.nix @@ -25,17 +25,14 @@ in { }; config = lib.mkIf cfg.enable { - fileSystems."${cfg.hostPath}" = { + homelab.nfsMounts."${cfg.hostPath}" = { device = "192.168.0.11:${remotePath}"; - fsType = "nfs"; - options = [ - permissionsOption + readOnly = permissionsOption == "ro"; + extraOptions = [ "auto" - "nfsvers=4.2" - "async" "soft" "rsize=1048576" "wsize=1048576" - "timeo=600" "retry=50" "retrans=2" "actimeo=1800" "lookupcache=all" - "_netdev" "nosuid" "tcp" + "timeo=600" "retrans=2" + "_netdev" ]; }; }; diff --git a/modules/services/actions/default.nix b/nixos/modules/services/actions/default.nix similarity index 100% rename from modules/services/actions/default.nix rename to nixos/modules/services/actions/default.nix diff --git a/modules/services/default.nix b/nixos/modules/services/default.nix similarity index 68% rename from modules/services/default.nix rename to nixos/modules/services/default.nix index f70bc54..ec5cb63 100644 --- a/modules/services/default.nix +++ b/nixos/modules/services/default.nix @@ -2,5 +2,6 @@ imports = [ ./actions ./openssh + ./hypervisor-gitops ]; } diff --git a/nixos/modules/services/hypervisor-gitops/default.nix b/nixos/modules/services/hypervisor-gitops/default.nix new file mode 100644 index 0000000..a7f5b7c --- /dev/null +++ b/nixos/modules/services/hypervisor-gitops/default.nix @@ -0,0 +1,118 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.homelab.services.hypervisor-gitops; + + hypervisorSyncScript = pkgs.writeShellApplication { + name = "hypervisor-sync"; + runtimeInputs = with pkgs; [ git opentofu coreutils ]; + text = builtins.readFile ../../../../scripts/hypervisor-sync.sh; + }; + + stagingSyncScript = pkgs.writeShellApplication { + name = "staging-sync"; + runtimeInputs = with pkgs; [ opentofu coreutils jq curl ]; + text = builtins.readFile ../../../../scripts/staging-sync.sh; + }; + + natsConsumerScript = pkgs.writeShellApplication { + name = "nats-consumer"; + runtimeInputs = with pkgs; [ natscli jq stagingSyncScript ]; + text = '' + set -euo pipefail + + NATS_URL=''${NATS_URL:-"nats://192.168.0.20:4222"} + + echo "Starting NATS JetStream consumer for staging env..." + + # Try to create stream and consumer if they don't exist + nats --server "$NATS_URL" stream add FORGEJO_EVENTS --subjects "forgejo.staging" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file -f || true + nats --server "$NATS_URL" consumer add FORGEJO_EVENTS STAGING --pull --ack explicit --filter forgejo.staging --deliver all -f || true + + echo "Listening for messages..." + while true; do + # We use a simple sub to pull messages. In a real environment, + # a dedicated Go/Python client is better for manual explicit acks. + # This will auto-ack upon receipt and pass to the staging script. + nats --server "$NATS_URL" sub "forgejo.staging" | awk '/\[#.*\]/{flag=1; next} flag' | staging-sync || true + done + ''; + }; +in { + options.homelab.services.hypervisor-gitops = { + enable = mkEnableOption "Hypervisor GitOps Service"; + + repoUrl = mkOption { + type = types.str; + description = "The URL of the git repository to pull"; + }; + + # TODO Replace with webhooks + pollInterval = mkOption { + type = types.str; + default = "hourly"; + description = "Systemd calendar event for polling interval"; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = with pkgs; [ + git + opentofu + natscli + jq + ]; + + systemd.services.hypervisor-gitops = { + description = "Hypervisor GitOps Polling Service"; + + # We need network access to reach Forgejo and Proxmox API + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + + serviceConfig = { + Type = "oneshot"; + User = "root"; # Needs root to read SOPS secrets potentially + + # We will create a state directory for the repo + StateDirectory = "hypervisor-gitops"; + WorkingDirectory = "/var/lib/hypervisor-gitops"; + + ExecStart = "${hypervisorSyncScript}/bin/hypervisor-sync ${cfg.repoUrl} opentofu/workload-layer/production"; + }; + }; + + systemd.timers.hypervisor-gitops = { + description = "Timer for Hypervisor GitOps Service"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.pollInterval; + Persistent = true; + }; + }; + + systemd.services.staging-sync = { + description = "Staging Environment NATS Consumer"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "simple"; + User = "root"; # Needs root to read SOPS secrets + Restart = "always"; + RestartSec = "10s"; + + StateDirectory = "hypervisor-gitops"; + WorkingDirectory = "/var/lib/hypervisor-gitops"; + + # We assume TRUENAS_API_KEY is provided via a sops EnvironmentFile + # EnvironmentFile = config.sops.secrets."truenas-api-key".path; + + ExecStart = "${natsConsumerScript}/bin/nats-consumer"; + }; + }; + }; +} diff --git a/modules/services/openssh/default.nix b/nixos/modules/services/openssh/default.nix similarity index 100% rename from modules/services/openssh/default.nix rename to nixos/modules/services/openssh/default.nix diff --git a/modules/virtualisation/containers/default.nix b/nixos/modules/virtualisation/containers/default.nix similarity index 100% rename from modules/virtualisation/containers/default.nix rename to nixos/modules/virtualisation/containers/default.nix diff --git a/modules/virtualisation/default.nix b/nixos/modules/virtualisation/default.nix similarity index 100% rename from modules/virtualisation/default.nix rename to nixos/modules/virtualisation/default.nix diff --git a/modules/virtualisation/guest/default.nix b/nixos/modules/virtualisation/guest/default.nix similarity index 100% rename from modules/virtualisation/guest/default.nix rename to nixos/modules/virtualisation/guest/default.nix diff --git a/users/admin/default.nix b/nixos/users/admin/default.nix similarity index 85% rename from users/admin/default.nix rename to nixos/users/admin/default.nix index dc01c81..65410ff 100644 --- a/users/admin/default.nix +++ b/nixos/users/admin/default.nix @@ -9,7 +9,7 @@ in { type = lib.types.listOf lib.types.str; default = [ # HomeLab > NixOS > admin > ssh - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGWIOOEqTy8cWKpENVbzD4p7bsQgQb/Dgpzk8i0dZ00T" + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIHOoTp+e6qWGn4Sco5CZ6G0zrX5NAQBpLlDVirncJ/HqAAAABHNzaDo=" ]; }; }; diff --git a/users/apps/default.nix b/nixos/users/apps/default.nix similarity index 100% rename from users/apps/default.nix rename to nixos/users/apps/default.nix diff --git a/users/backup/default.nix b/nixos/users/backup/default.nix similarity index 100% rename from users/backup/default.nix rename to nixos/users/backup/default.nix diff --git a/users/default.nix b/nixos/users/default.nix similarity index 100% rename from users/default.nix rename to nixos/users/default.nix diff --git a/users/deploy/default.nix b/nixos/users/deploy/default.nix similarity index 100% rename from users/deploy/default.nix rename to nixos/users/deploy/default.nix diff --git a/users/media/default.nix b/nixos/users/media/default.nix similarity index 100% rename from users/media/default.nix rename to nixos/users/media/default.nix diff --git a/opentofu/host-layer/main.tf b/opentofu/host-layer/main.tf new file mode 100644 index 0000000..681858e --- /dev/null +++ b/opentofu/host-layer/main.tf @@ -0,0 +1,54 @@ +terraform { + # This state is dedicated purely to the physical host configuration. + # No providers are required since we use null_resource and ssh. +} + +resource "null_resource" "bare_metal_setup" { + triggers = { + node = var.node_ip + data_disk_id = var.data_disk_id + pool_name = var.zfs_pool_name + } + + connection { + type = "ssh" + user = var.ssh_user + host = var.node_ip + private_key = var.ssh_private_key != "" ? var.ssh_private_key : null + agent = var.ssh_private_key == "" ? true : false + } + + provisioner "remote-exec" { + inline = [ + "set -euo pipefail", + + + "echo '==> Setting laptop lid switch to ignore (prevents sleeping when closed)...'", + "sed -i 's/^#\\?HandleLidSwitch=.*/HandleLidSwitch=ignore/' /etc/systemd/logind.conf", + "systemctl restart systemd-logind", + + "echo '==> Applying NIC offloading fixes...'", + "cat <<'EOF' > /etc/systemd/system/nic-offload-fix.service", + "[Unit]", + "Description=Disable NIC offloading (TSO/GRO/GSO) for physical interfaces", + "After=network-online.target", + "", + "[Service]", + "Type=oneshot", + "ExecStart=/bin/bash -c 'for dev in /sys/class/net/*; do if [ \"$(basename \"$dev\")\" != \"lo\" ] && [[ ! \"$(basename \"$dev\")\" =~ ^(vmbr|veth|fwbr|tap|bonding) ]]; then /usr/sbin/ethtool -K \"$(basename \"$dev\")\" tso off gso off gro off || true; fi; done'", + "RemainAfterExit=yes", + "", + "[Install]", + "WantedBy=multi-user.target", + "EOF", + "systemctl daemon-reload", + "systemctl enable --now nic-offload-fix.service || true", + + "echo '==> Setting up ZFS pool and Proxmox storage...'", + "zpool list ${var.zfs_pool_name} || zpool create -f ${var.zfs_pool_name} /dev/disk/by-id/${var.data_disk_id}", + "pvesm status -storage ${var.zfs_pool_name} || pvesm add zfspool ${var.zfs_pool_name} --pool ${var.zfs_pool_name} --content images,rootdir", + + "echo '==> Host layer configuration complete!'" + ] + } +} diff --git a/opentofu/host-layer/variables.tf b/opentofu/host-layer/variables.tf new file mode 100644 index 0000000..6fa74c3 --- /dev/null +++ b/opentofu/host-layer/variables.tf @@ -0,0 +1,27 @@ +variable "node_ip" { + type = string + description = "The IP address or hostname of the Proxmox node to configure" +} + +variable "ssh_user" { + type = string + default = "root" + description = "The SSH user to connect as" +} + +variable "ssh_private_key" { + type = string + default = "" + description = "The SSH private key content (if not using ssh-agent)" +} + +variable "data_disk_id" { + type = string + description = "The persistent block device ID for the data disk (e.g. wwn-0x500...)" +} + +variable "zfs_pool_name" { + type = string + default = "data" + description = "The name of the ZFS pool to create on the data disk" +} diff --git a/opentofu/modules/proxmox-node/main.tf b/opentofu/modules/proxmox-node/main.tf new file mode 100644 index 0000000..8588875 --- /dev/null +++ b/opentofu/modules/proxmox-node/main.tf @@ -0,0 +1,89 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.61.0" + } + } +} + + + +# Resource Pools + +# Creates logical groups for organizing and securing VMs. +resource "proxmox_virtual_environment_pool" "core" { + pool_id = "core" + comment = "Critical, public-facing services (reverse proxy, DNS)" +} + +resource "proxmox_virtual_environment_pool" "production" { + pool_id = "production" + comment = "Stable services with backup strategies" +} + +resource "proxmox_virtual_environment_pool" "dev" { + pool_id = "dev" + comment = "Persistent but non-production services, run 24/7" +} + +resource "proxmox_virtual_environment_pool" "staging" { + pool_id = "staging" + comment = "Ephemeral test VMs managed by Pull-Based PR webhooks" +} + +# API Tokens for Control Center + +# Creates a restricted user for the Control Center VM to manage the cluster. +resource "proxmox_virtual_environment_role" "control_center_role" { + role_id = "ControlCenter" + + privileges = [ + "VM.Allocate", + "VM.Audit", + "VM.Clone", + "VM.Config.CDROM", + "VM.Config.CPU", + "VM.Config.Disk", + "VM.Config.HWType", + "VM.Config.Memory", + "VM.Config.Network", + "VM.Config.Options", + "VM.Monitor", + "VM.PowerMgmt", + "Datastore.AllocateSpace", + "Datastore.Audit", + "SDN.Use" + ] +} + +resource "proxmox_virtual_environment_user" "control_center_user" { + user_id = "control-center@pve" + comment = "Managed by OpenTofu - Used by Control Center for GitOps" +} + +# Grant full VM access to the control center +resource "proxmox_virtual_environment_acl" "control_center_vms" { + user_id = proxmox_virtual_environment_user.control_center_user.user_id + role_id = proxmox_virtual_environment_role.control_center_role.role_id + path = "/vms" +} + +# Grant datastore access +resource "proxmox_virtual_environment_acl" "control_center_storage" { + user_id = proxmox_virtual_environment_user.control_center_user.user_id + role_id = proxmox_virtual_environment_role.control_center_role.role_id + path = "/storage/${var.zfs_pool_name}" +} + +resource "proxmox_virtual_environment_user_token" "control_center_token" { + user_id = proxmox_virtual_environment_user.control_center_user.user_id + token_name = "gitops" + comment = "Stored only on the Control Center VM" +} + +output "control_center_api_token" { + value = proxmox_virtual_environment_user_token.control_center_token.value + sensitive = true + description = "The secret API token for control-center@pve. This will be injected into the Control Center SOPS config." +} diff --git a/opentofu/modules/proxmox-node/variables.tf b/opentofu/modules/proxmox-node/variables.tf new file mode 100644 index 0000000..25b521a --- /dev/null +++ b/opentofu/modules/proxmox-node/variables.tf @@ -0,0 +1,11 @@ +variable "node_name" { + type = string + description = "The name of the Proxmox node (e.g. pve)" +} + + +variable "zfs_pool_name" { + type = string + default = "data" + description = "The name of the ZFS pool to create on the data disk" +} diff --git a/opentofu/workload-layer/production/main.tf b/opentofu/workload-layer/production/main.tf new file mode 100644 index 0000000..4d9a22d --- /dev/null +++ b/opentofu/workload-layer/production/main.tf @@ -0,0 +1,80 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.61.0" + } + } +} + +provider "proxmox" { + # Endpoint and credentials will be picked up from environment variables + # or passed via the bootstrap script. + # PROXMOX_VE_ENDPOINT + # PROXMOX_VE_USERNAME + # PROXMOX_VE_PASSWORD + # PROXMOX_VE_INSECURE=true +} + +module "proxmox_node" { + source = "../../modules/proxmox-node" + node_name = "mikoshi" + zfs_pool_name = "data" +} + +output "control_center_api_token" { + value = module.proxmox_node.control_center_api_token + sensitive = true + description = "The secret API token for control-center@pve." +} + +resource "proxmox_virtual_environment_vm" "control_center" { + depends_on = [module.proxmox_node] + + name = "izanagi" + description = "Managed by OpenTofu - GitOps Control Center" + tags = ["infrastructure", "gitops"] + node_name = "mikoshi" + vm_id = 100001000 + + on_boot = true + + pool_id = "core" + + cpu { + cores = 2 + type = "x86-64-v2-AES" + } + + memory { + dedicated = 2048 + } + + agent { + enabled = true + } + + network_device { + bridge = "vmbr0" + } + + clone { + vm_id = 9000 + full = true + } + + # Cloud-Init for initial SSH access and SOPS age key injection + initialization { + ip_config { + ipv4 { + address = "dhcp" + } + } + user_account { + username = "gh0st" + keys = [ + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIHOoTp+e6qWGn4Sco5CZ6G0zrX5NAQBpLlDVirncJ/HqAAAABHNzaDo=" + ] + } + } +} diff --git a/opentofu/workload-layer/staging/main.tf b/opentofu/workload-layer/staging/main.tf new file mode 100644 index 0000000..9538523 --- /dev/null +++ b/opentofu/workload-layer/staging/main.tf @@ -0,0 +1,131 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.61.0" + } + truenas = { + source = "deevus/truenas" + version = "~> 0.1.0" + } + } +} + +variable "vm_id" { + description = "The ID of the VM to create (should be unique per PR)" + type = number +} + +variable "pr_number" { + description = "The Pull Request number for this staging environment" + type = string +} + +variable "truenas_snapshot_id" { + description = "The ID of the TrueNAS snapshot to clone" + type = string +} + +provider "proxmox" { + # Relies on PROXMOX_VE_ENDPOINT and PROXMOX_VE_API_TOKEN environment variables +} + +provider "truenas" { + # Relies on TRUENAS_API_KEY and TRUENAS_BASE_URL environment variables +} + +resource "truenas_dataset" "staging_clone" { + pool = "tank" + path = "production/staging-pr-${var.pr_number}" + snapshot_id = var.truenas_snapshot_id +} + +resource "proxmox_virtual_environment_vm" "staging_vm" { + name = "staging-pr-${var.pr_number}" + description = "Ephemeral staging environment for PR #${var.pr_number}" + node_name = "pve" + vm_id = var.vm_id + + # Clone from the latest golden image template + clone { + vm_id = 9000 + full = true + } + + agent { + enabled = true + } + + cpu { + cores = 2 + } + + memory { + dedicated = 2048 + } + + network_device { + bridge = "vmbr0" + firewall = true + } + + # Cloud-Init configuration to inject the staging age key and set up networking + + initialization { + ip_config { + ipv4 { + address = "dhcp" + } + } + + user_data_file_id = "local:snippets/staging-key.yaml" + } +} + + + +resource "proxmox_virtual_environment_firewall_options" "staging_vm_fw_options" { + vm_id = proxmox_virtual_environment_vm.staging_vm.vm_id + node_name = proxmox_virtual_environment_vm.staging_vm.node_name + enable = true + policy_in = "ACCEPT" + policy_out = "DROP" +} + +resource "proxmox_virtual_environment_firewall_rules" "staging_vm_rules" { + vm_id = proxmox_virtual_environment_vm.staging_vm.vm_id + node_name = proxmox_virtual_environment_vm.staging_vm.node_name + + rule { + action = "ACCEPT" + type = "out" + dest = "192.168.0.11" + comment = "Allow traffic to TrueNAS" + } + + rule { + action = "ACCEPT" + type = "out" + dest = "192.168.0.1" + comment = "Allow traffic to Gateway/DNS" + } + + rule { + action = "DROP" + type = "out" + dest = "192.168.0.0/24" + comment = "Drop traffic to local homelab" + } + + rule { + action = "ACCEPT" + type = "out" + dest = "0.0.0.0/0" + comment = "Allow outbound internet traffic" + } +} + +output "staging_vm_ip" { + value = proxmox_virtual_environment_vm.staging_vm.ipv4_addresses[1][0] # Adjust index based on actual returned interfaces + description = "The IP address of the newly spun up staging VM." +} diff --git a/scripts/apply-host-layer.sh b/scripts/apply-host-layer.sh new file mode 100755 index 0000000..46c69ce --- /dev/null +++ b/scripts/apply-host-layer.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script runs the host-layer OpenTofu configuration using a Docker container, +# keeping your laptop and the Proxmox node clean of dependencies. + +echo "Running OpenTofu for host-layer..." + +docker run --rm -it \ + -v "$(pwd)":/workspace \ + -w /workspace/opentofu/host-layer \ + -v "$HOME/.ssh:/root/.ssh:ro" \ + ghcr.io/opentofu/opentofu:latest init -upgrade + +docker run --rm -it \ + -v "$(pwd)":/workspace \ + -w /workspace/opentofu/host-layer \ + -v "$HOME/.ssh:/root/.ssh:ro" \ + ghcr.io/opentofu/opentofu:latest apply diff --git a/scripts/hypervisor-sync.sh b/scripts/hypervisor-sync.sh new file mode 100644 index 0000000..809baa7 --- /dev/null +++ b/scripts/hypervisor-sync.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# This script pulls the latest changes from the Git repository +# and runs OpenTofu to provision the hypervisor state. +# Usage: ./hypervisor-sync.sh + +REPO_URL=${1:-"https://git.depeuter.dev/Bos55/nix-config.git"} +OPENTOFU_DIR=${2:-"opentofu/workload-layer/production"} + +echo "Starting Hypervisor GitOps sync..." + +if [ ! -d "nix-config" ]; then + echo "Cloning repository from $REPO_URL..." + git clone "$REPO_URL" nix-config +fi + +cd nix-config || exit + +git fetch origin v2 + +LOCAL=$(git rev-parse HEAD) +REMOTE=$(git rev-parse origin/v2) + +if [ "$LOCAL" = "$REMOTE" ]; then + echo "Already up to date. Nothing to do." + exit 0 +fi + +echo "Changes detected. Updating from $LOCAL to $REMOTE..." +git reset --hard origin/v2 + +echo "Applying OpenTofu changes in $OPENTOFU_DIR..." +cd "$OPENTOFU_DIR" || exit + +tofu init -upgrade + +tofu apply -auto-approve + +echo "Hypervisor GitOps sync completed successfully." diff --git a/scripts/nixos-sync.sh b/scripts/nixos-sync.sh new file mode 100644 index 0000000..18f0c0f --- /dev/null +++ b/scripts/nixos-sync.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# This script checks the remote Git repository for changes +# and triggers a nixos-rebuild if a new commit is found. +# Usage: ./nixos-sync.sh + +set -euo pipefail + +REPO_URL=${1:-"https://git.depeuter.dev/Bos55/nix-config.git"} +BRANCH=${2:-"main"} + +echo "Checking remote hash for $REPO_URL branch $BRANCH..." + +# Fetch remote hash, fallback to unknown if it fails +REMOTE_HASH=$(git ls-remote "$REPO_URL" "refs/heads/$BRANCH" | awk '{print $1}' || true) + +if [ -z "$REMOTE_HASH" ]; then + echo "WARNING: Could not fetch remote hash. Forcing rebuild to be safe." + REMOTE_HASH="unknown_remote" +fi + +LOCAL_HASH="unknown_local" +if [ -f /run/current-system/configurationRevision ]; then + LOCAL_HASH=$(cat /run/current-system/configurationRevision) +fi + +echo "Remote hash: $REMOTE_HASH" +echo "Local hash: $LOCAL_HASH" + +if [ "$REMOTE_HASH" = "$LOCAL_HASH" ] && [ "$REMOTE_HASH" != "unknown_remote" ] && [ "$LOCAL_HASH" != "unknown" ]; then + echo "Hashes match. No update needed." + exit 0 +fi + +echo "Hashes differ or unknown. Triggering nixos-rebuild..." + +# Trigger the build and switch +nixos-rebuild switch --flake "git+$REPO_URL?dir=nixos&ref=$BRANCH" + +echo "Update successful." diff --git a/scripts/staging-sync.sh b/scripts/staging-sync.sh new file mode 100644 index 0000000..a62879e --- /dev/null +++ b/scripts/staging-sync.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +# This script is triggered by the NATS JetStream consumer on the Control Center. +# It reads a Forgejo webhook JSON payload from STDIN and orchestrates the +# Staging VM OpenTofu lifecycle. + +set -euo pipefail + +# Read JSON payload from STDIN +PAYLOAD=$(cat) + +# Extract fields using jq +ACTION=$(echo "$PAYLOAD" | jq -r '.action // empty') +PR_NUMBER=$(echo "$PAYLOAD" | jq -r '.pull_request.number // empty') + +if [ -z "$ACTION" ] || [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" == "null" ]; then + printf "Invalid or missing action/pr_number in payload. Exiting.\n" + exit 0 +fi + +# We only care about PR events +if [[ "$ACTION" != "opened" && "$ACTION" != "reopened" && "$ACTION" != "synchronized" && "$ACTION" != "closed" ]]; then + printf "Ignoring PR action: %s\n" "$ACTION" + exit 0 +fi + +# Ensure workspace directory exists for this PR +WORKSPACE="/var/lib/hypervisor-gitops/staging-pr-${PR_NUMBER}" +OPENTOFU_SRC="/var/lib/hypervisor-gitops/nix-config/opentofu/workload-layer/staging" + +# TrueNAS API variables +TRUENAS_URL=${TRUENAS_URL:-"https://192.168.0.11"} +TRUENAS_API_KEY=${TRUENAS_API_KEY:-""} +ZFS_DATASET=${ZFS_DATASET:-"tank/production"} + +get_latest_snapshot() { + # Fetch the latest snapshot for the dataset from TrueNAS API + # Assumes TRUENAS_API_KEY is exported in the environment by SOPS + curl -sS -k -X GET \ + -H "Authorization: Bearer ${TRUENAS_API_KEY}" \ + -H "Accept: application/json" \ + "${TRUENAS_URL}/api/v2.0/zfs/snapshot?id~=${ZFS_DATASET}%25&limit=1&sort=-creation" | jq -r '.[0].id' +} + +printf "Processing PR #%s (Action: %s)\n" "$PR_NUMBER" "$ACTION" + +if [[ "$ACTION" == "closed" ]]; then + if [ ! -d "$WORKSPACE" ]; then + printf "Workspace %s does not exist. Nothing to destroy.\n" "$WORKSPACE" + exit 0 + fi + printf "Destroying Staging Environment for PR #%s...\n" "$PR_NUMBER" + cd "$WORKSPACE" + tofu destroy -var="pr_number=${PR_NUMBER}" -auto-approve + + # Cleanup + cd / + rm -rf "$WORKSPACE" + printf "Staging Environment Destroyed.\n" + exit 0 +fi + +# Provisioning (opened, reopened, synchronized) +printf "Setting up Staging Environment for PR #%s...\n" "$PR_NUMBER" + +if [ ! -d "$WORKSPACE" ]; then + mkdir -p "$WORKSPACE" + cp -r "$OPENTOFU_SRC"/* "$WORKSPACE"/ +fi + +cd "$WORKSPACE" + +# Fetch latest TrueNAS snapshot dynamically +LATEST_SNAPSHOT=$(get_latest_snapshot) + +if [ -z "$LATEST_SNAPSHOT" ] || [ "$LATEST_SNAPSHOT" == "null" ]; then + printf "Failed to retrieve the latest TrueNAS snapshot. Aborting.\n" + exit 1 +fi + +printf "Latest Snapshot ID: %s\n" "$LATEST_SNAPSHOT" + +# Apply OpenTofu +tofu init -upgrade +tofu apply -var="pr_number=${PR_NUMBER}" -var="truenas_snapshot_id=${LATEST_SNAPSHOT}" -auto-approve + +printf "Staging Environment Provisioned successfully.\n" diff --git a/scripts/truenas-rbac-setup.sh b/scripts/truenas-rbac-setup.sh new file mode 100755 index 0000000..bca71bf --- /dev/null +++ b/scripts/truenas-rbac-setup.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script automates the creation of a restricted TrueNAS user (forgejo-ci) +# and assigns it a custom privilege role strictly limited to ZFS cloning/snapshots. + +echo "==========================================" +echo " TrueNAS RBAC Setup for CI/CD" +echo "==========================================" +echo "This script will create a custom Privilege Role and a Restricted User." +echo "" + +read -p "Enter your TrueNAS IP (e.g., 192.168.0.11): " TRUENAS_IP +read -s -p "Enter your current TrueNAS Admin Token (root): " ADMIN_TOKEN +echo "" +read -p "Enter a password for the new 'forgejo-ci' user: " CI_PASSWORD + +BASE_URL="http://${TRUENAS_IP}/api/v2.0" +HEADERS=( + "-H" "Authorization: Bearer ${ADMIN_TOKEN}" + "-H" "Content-Type: application/json" +) + +echo "" +echo "1. Creating Custom Privilege (ci-runner-role)..." +# In TrueNAS SCALE, we create a privilege that allows specific methods +PRIV_PAYLOAD=$(cat < Privilege created successfully." +elif [ "$HTTP_STATUS" -eq 409 ] || [ "$HTTP_STATUS" -eq 422 ]; then + echo " -> Privilege already exists or validation failed (code ${HTTP_STATUS}). Skipping." +else + echo " -> Warning: Privilege creation returned HTTP ${HTTP_STATUS}. (Your TrueNAS version might handle RBAC differently)." +fi + +echo "2. Creating Restricted User (forgejo-ci)..." +USER_PAYLOAD=$(cat < User created successfully." +elif [ "$HTTP_STATUS" -eq 409 ] || [ "$HTTP_STATUS" -eq 422 ]; then + echo " -> User already exists. Skipping." +else + echo " -> Warning: User creation returned HTTP ${HTTP_STATUS}." + cat /tmp/truenas_user.json +fi + +echo "" +echo "==========================================" +echo " Setup Complete (or mostly complete)!" +echo "==========================================" +echo "Because TrueNAS prevents root from generating API tokens for other users," +echo "you must complete the final step manually:" +echo "" +echo "1. Log into the TrueNAS Web UI at http://${TRUENAS_IP}" +echo "2. If the script failed to attach the privilege automatically, go to Credentials > Local Users," +echo " edit 'forgejo-ci', and assign it the ZFS roles." +echo "3. Log in as 'forgejo-ci' (or use the API Keys menu as Admin to generate a key for that user)." +echo "4. Copy the newly generated token." +echo "5. Update the TRUENAS_API_KEY secret in your Forgejo repository." +echo "==========================================" diff --git a/scripts/truenas-staging-clone.sh b/scripts/truenas-staging-clone.sh new file mode 100755 index 0000000..1659303 --- /dev/null +++ b/scripts/truenas-staging-clone.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script creates a ZFS snapshot of a production dataset and clones it for a staging environment. +# Required environment variables: +# TRUENAS_IP: The IP address of the TrueNAS scale instance +# TRUENAS_API_KEY: The API token for TrueNAS +# POOL_NAME: The name of the ZFS pool (e.g., "tank") +# SOURCE_DATASET: The name of the production dataset (e.g., "apps/jellyfin") +# PR_NUMBER: The Pull Request number + +if [[ -z "${TRUENAS_IP:-}" || -z "${TRUENAS_API_KEY:-}" || -z "${POOL_NAME:-}" || -z "${SOURCE_DATASET:-}" || -z "${PR_NUMBER:-}" ]]; then + echo "Error: Missing required environment variables." + exit 1 +fi + +BASE_URL="http://${TRUENAS_IP}/api/v2.0" +HEADERS=( + "-H" "Authorization: Bearer ${TRUENAS_API_KEY}" + "-H" "Content-Type: application/json" +) + +DATASET_ID="${POOL_NAME}/${SOURCE_DATASET}" +SNAPSHOT_NAME="pr-${PR_NUMBER}-base" +CLONE_DATASET_NAME="${SOURCE_DATASET}-pr-${PR_NUMBER}" + +echo "1. Creating snapshot of ${DATASET_ID} @ ${SNAPSHOT_NAME}..." +curl -s -X POST "${BASE_URL}/zfs/snapshot" "${HEADERS[@]}" \ + -d "{\"dataset\": \"${DATASET_ID}\", \"name\": \"${SNAPSHOT_NAME}\"}" > /dev/null + +echo "2. Cloning snapshot to ${POOL_NAME}/${CLONE_DATASET_NAME}..." +curl -s -X POST "${BASE_URL}/zfs/snapshot/clone" "${HEADERS[@]}" \ + -d "{\"snapshot\": \"${DATASET_ID}@${SNAPSHOT_NAME}\", \"dataset_dst\": \"${CLONE_DATASET_NAME}\"}" > /dev/null + +echo "Staging dataset cloned successfully." diff --git a/scripts/truenas-staging-teardown.sh b/scripts/truenas-staging-teardown.sh new file mode 100755 index 0000000..9b68f26 --- /dev/null +++ b/scripts/truenas-staging-teardown.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script destroys the ephemeral staging ZFS clone and the base snapshot. +# Required environment variables: +# TRUENAS_IP: The IP address of the TrueNAS scale instance +# TRUENAS_API_KEY: The API token for TrueNAS +# POOL_NAME: The name of the ZFS pool (e.g., "tank") +# SOURCE_DATASET: The name of the production dataset (e.g., "apps/jellyfin") +# PR_NUMBER: The Pull Request number + +if [[ -z "${TRUENAS_IP:-}" || -z "${TRUENAS_API_KEY:-}" || -z "${POOL_NAME:-}" || -z "${SOURCE_DATASET:-}" || -z "${PR_NUMBER:-}" ]]; then + echo "Error: Missing required environment variables." + exit 1 +fi + +BASE_URL="http://${TRUENAS_IP}/api/v2.0" +HEADERS=( + "-H" "Authorization: Bearer ${TRUENAS_API_KEY}" + "-H" "Content-Type: application/json" +) + +DATASET_ID="${POOL_NAME}/${SOURCE_DATASET}" +SNAPSHOT_NAME="pr-${PR_NUMBER}-base" +CLONE_DATASET_ID="${POOL_NAME}/${SOURCE_DATASET}-pr-${PR_NUMBER}" + +# TrueNAS API requires the ID to be URL-encoded for dataset deletion +# URL encoding helper (replace / with %2F) +ENCODED_CLONE_ID="${CLONE_DATASET_ID//\//%2F}" +ENCODED_SNAPSHOT_ID="${DATASET_ID}@${SNAPSHOT_NAME}" +ENCODED_SNAPSHOT_ID="${ENCODED_SNAPSHOT_ID//\//%2F}" + +echo "1. Destroying staging clone ${CLONE_DATASET_ID}..." +curl -s -X DELETE "${BASE_URL}/zfs/dataset/id/${ENCODED_CLONE_ID}" "${HEADERS[@]}" > /dev/null + +echo "2. Destroying base snapshot ${DATASET_ID}@${SNAPSHOT_NAME}..." +curl -s -X DELETE "${BASE_URL}/zfs/snapshot/id/${ENCODED_SNAPSHOT_ID}" "${HEADERS[@]}" > /dev/null + +echo "Staging dataset and snapshot cleaned up successfully." diff --git a/secrets/secrets.yaml b/secrets/secrets.yaml deleted file mode 100644 index e17dab1..0000000 --- a/secrets/secrets.yaml +++ /dev/null @@ -1,19 +0,0 @@ -users: - admin: - authorized_keys: - NixOS: ENC[AES256_GCM,data:sj2hkUkWp628KuXp+AnncLdawHpxb9fH1ZHnIisP0x9Tght9+/X2sWHpuMSeqi2i/R8B+Wgte66QkuwAOB0j+oB9N+66EhehmWZlK5hD/22p,iv:z18U+LvAQgPDfBBewE3lJmWZd0NGCPwJIe/h3tupuZc=,tag:ZJar3spO66JbDXygdTHh2w==,type:str] -sops: - age: - - recipient: age1qzutny0mqpcccqw6myyfntu6wcskruu9ghzvt6r4te7afkqwnguq05ex37 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjUSt2REk2Mmd0bk9ubjJk - dXFiY2JNR1dyZW9qTUdzaWZhY3c3amVwQzA0CkZHNVpZVjhsWXhVQVNaR0xONzhh - Y0lQaWNaNmpYYVdrRnZIZUhvUFUzcWMKLS0tIDAvSmF0VmpxcnZEQStXUjNCUE5Z - RnA2Lzk2WHFxOEh6dHN0aGhVSVpLTW8KA7IOvGDMBtgo4pe0Sw3Lol243xCDAJ4i - PhcJFiUObVRFZN7ISlULnOlTO3pT9jWvvmC5rDZWId3PQ8qjPvnOUg== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-04T17:33:22Z" - mac: ENC[AES256_GCM,data:I7I7uDFEWfw9+4KROtjHMVhaxYrVK5QmLfFZShSajF0A2Zxu9lg+fDGiMHk40JC5zD31P70QS/ipye1mBGQbCbLEA7uBUhNzZ7G1g58cIXF6vSGmt0fovm0MVSxEJ44r05fx6uT4OJu5BYVxYSlG84gTj9rCFXxxcBJMrh+6yaI=,iv:c1vudsp9bg0Pc2ddRyvWn6Tf0LhqNuEjxG9D4PpHqxs=,tag:K/1PSHhrTdsNPcPmRv/2Ew==,type:str] - unencrypted_suffix: _unencrypted - version: 3.10.2