ansible

ansible/README.md

@@ -179,7 +179,7 @@ git clone git@git.asxp.io:asxpi/infrastructure.git
 ansible-playbook playbooks/wireguard.yaml
 ansible-playbook playbooks/wireguard-client.yaml
 
-# 2. Harden SSH
+# 2. Harden SSH (Switch to WireGuard IP's in hosts.yaml before applying)
 ansible-playbook playbooks/ssh-harden.yaml
 
 # 3. Deploy reverse proxy

ansible/inventory/group_vars/all.yaml

@@ -14,7 +14,8 @@ forgejo_ssh_port: 2222  # External port (container uses 22)
 forgejo_runner_version: "11"
 forgejo_runner_capacity: 1
 forgejo_runner_timeout: 3h
+forgejo_runner_token: "***REDACTED***"
 
 # Backup configuration
-backup_dest_dir: /tmp/forgejo-backups
-backup_old_server: 10.0.0.1
+backup_dest_dir: /opt/forgejo-backup
+backup_old_server: asxpio-old

ansible/inventory/host_vars/asxpio-ops.yaml

@@ -45,3 +45,11 @@ wireguard_dns_servers:
 # SSH Hardening Configuration
 # Allow public SSH access for Forgejo git operations
 ssh_allow_public_git: true
+ssh_git_user: git
+
+# Forgejo secrets (from old server)
+forgejo_secret_key: "***REDACTED***"
+forgejo_internal_token: "***REDACTED***"
+forgejo_jwt_secret: "***REDACTED***"
+forgejo_lfs_jwt_secret: "***REDACTED***"
+letsencrypt_email: "me@asxp.io"

ansible/inventory/hosts.yaml

@@ -18,7 +18,8 @@ all:
         ops_servers:
           hosts:
             asxpio-ops:
-              ansible_host: 10.20.0.10 #65.109.14.216
+              ansible_host: 10.20.0.10
+              #ansible_host: 65.109.14.216
               server_role: ops
               server_type: cx22
               services:
@@ -32,7 +33,8 @@ all:
         services_servers:
           hosts:
             asxpio-services:
-              ansible_host: 10.20.0.20 #95.217.15.42
+              ansible_host: 10.20.0.20
+              #ansible_host: 95.217.15.42
               server_role: services
               server_type: cx43
               services:
@@ -43,3 +45,10 @@ all:
                 - factorio
                 - prosody
                 - coturn
+
+        old_servers:
+          hosts:
+            asxpio-old:
+              ansible_host: asxp.io
+              server_role: legacy
+              ansible_user: asxpi

ansible/playbooks/backup-forgejo.yaml

@@ -1,8 +1,6 @@
 ---
 - name: Backup Forgejo from old server
-  hosts: "{{ backup_old_server }}"
+  hosts: asxpio-old
   become: true
-  vars:
-    ansible_host: "{{ backup_old_server }}"
   roles:
     - forgejo_backup

ansible/playbooks/ssh-harden.yaml

@@ -1,6 +1,6 @@
 ---
 - name: Harden SSH configuration
-  hosts: asxpio
+  hosts: asxpio:!old_servers
   become: true
   roles:
     - ssh_hardening

ansible/roles/forgejo/handlers/main.yaml

@@ -3,3 +3,8 @@
   community.docker.docker_compose_v2:
     project_src: /opt/forgejo
     state: restarted
+
+- name: Reload SSH daemon
+  systemd:
+    name: sshd
+    state: reloaded

ansible/roles/forgejo/tasks/main.yaml

@@ -1,4 +1,7 @@
 ---
+- name: Configure SSH passthrough
+  import_tasks: ssh-passthrough.yaml
+
 - name: Create Forgejo directories
   file:
     path: "{{ item }}"

ansible/roles/forgejo/tasks/ssh-passthrough.yaml

@@ -0,0 +1,45 @@
+---
+- name: Create git user on host
+  user:
+    name: git
+    comment: "Forgejo Git User"
+    home: /var/lib/git
+    shell: /usr/local/bin/forgejo-shell
+    system: yes
+    create_home: yes
+
+- name: Deploy forgejo-shell wrapper script
+  template:
+    src: forgejo-shell.j2
+    dest: /usr/local/bin/forgejo-shell
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Create .ssh directory for git user
+  file:
+    path: /var/lib/git/.ssh
+    state: directory
+    owner: git
+    group: git
+    mode: '0700'
+
+- name: Deploy SSH daemon configuration for Forgejo passthrough
+  template:
+    src: sshd-forgejo.conf.j2
+    dest: /etc/ssh/sshd_config.d/forgejo.conf
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Reload SSH daemon
+
+- name: Ensure docker group exists
+  group:
+    name: docker
+    state: present
+
+- name: Add git user to docker group
+  user:
+    name: git
+    groups: docker
+    append: yes

ansible/roles/forgejo/templates/app.ini.j2

@@ -8,15 +8,15 @@ ROOT_URL = https://{{ forgejo_domain }}/
 HTTP_PORT = 3000
 DISABLE_SSH = false
 SSH_DOMAIN = {{ forgejo_domain }}
-SSH_PORT = {{ forgejo_ssh_port }}
+SSH_PORT = 22
 SSH_LISTEN_PORT = 22
-START_SSH_SERVER = true
+START_SSH_SERVER = false
 LFS_START_SERVER = true
 OFFLINE_MODE = false
 
 [database]
 DB_TYPE = sqlite3
-PATH = /data/gitea/gitea.db
+PATH = /data/gitea.db
 LOG_SQL = false
 
 [repository]

ansible/roles/forgejo/templates/docker-compose.yml.j2

@@ -15,7 +15,6 @@ services:
       - /etc/localtime:/etc/localtime:ro
     ports:
       - "{{ forgejo_http_port }}:3000"
-      - "{{ forgejo_ssh_port }}:22"
     networks:
       - {{ traefik_network }}
     labels:

ansible/roles/forgejo/templates/forgejo-shell.j2

@@ -0,0 +1,4 @@
+#!/bin/sh
+# Forgejo SSH passthrough wrapper
+# Relays SSH commands to Forgejo container
+docker exec -i -u git --env SSH_ORIGINAL_COMMAND="$SSH_ORIGINAL_COMMAND" forgejo sh "$@"

ansible/roles/forgejo/templates/sshd-forgejo.conf.j2

@@ -0,0 +1,6 @@
+# Forgejo SSH passthrough configuration
+# Delegates SSH key authentication for git user to Forgejo container
+
+Match User git
+    AuthorizedKeysCommand /usr/bin/docker exec -i -u git forgejo /usr/local/bin/forgejo keys -c /data/gitea/conf/app.ini -e git -u %u -t %t -k %k
+    AuthorizedKeysCommandUser git

ansible/roles/forgejo_backup/tasks/main.yaml

@@ -1,79 +1,67 @@
 ---
-- name: Create local backup directory
+- name: Create backup directory on ops server
   file:
     path: "{{ backup_dest_dir }}"
     state: directory
     mode: '0755'
-  delegate_to: localhost
-  become: false
+    owner: asxpi
+    group: asxpi
+  delegate_to: asxpio-ops
+  become: true
 
-- name: Check if Forgejo service is running on old server
-  systemd:
-    name: forgejo
-  register: forgejo_service
-
-- name: Stop Forgejo service temporarily for backup
-  systemd:
-    name: forgejo
-    state: stopped
-  when: forgejo_service.status.ActiveState == "active"
-
-- name: Create Forgejo dump backup
-  command: >
-    /usr/bin/forgejo dump
-    -c /etc/forgejo/app.ini
-    -w /srv/gitea
-    --tempdir /tmp
-  args:
-    chdir: /srv/gitea
-  register: dump_output
-
-- name: Get dump filename
+- name: Find existing Forgejo dump files on old server
   find:
     paths: /srv/gitea
-    patterns: "gitea-dump-*.zip"
-  register: dump_file
+    patterns: "*.zip"
+    file_type: file
+  register: existing_dumps
 
-- name: Create additional backup archive
+- name: Create backup archive from old server
   archive:
     path:
       - /srv/gitea/gitea.db
-      - /srv/gitea/data
-      - /var/lib/forgejo/.ssh
-      - /var/lib/forgejo-runner/.runner
-    dest: /tmp/forgejo-additional-{{ ansible_date_time.date }}.tar.gz
+      - /srv/gitea/data/avatars
+      - /srv/gitea/data/repo-avatars
+      - /srv/gitea/data/repo-archive
+      - /srv/gitea/data/attachments
+      - /srv/gitea/data/gitea-repositories
+    dest: /tmp/forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz
     format: gz
-  ignore_errors: true
+  become: true
 
-- name: Fetch dump file to local machine
-  fetch:
-    src: "{{ dump_file.files[0].path }}"
-    dest: "{{ backup_dest_dir }}/"
-    flat: true
-  when: dump_file.files | length > 0
-
-- name: Fetch additional backup to local machine
-  fetch:
-    src: /tmp/forgejo-additional-{{ ansible_date_time.date }}.tar.gz
-    dest: "{{ backup_dest_dir }}/"
-    flat: true
-  ignore_errors: true
-
-- name: Start Forgejo service
-  systemd:
-    name: forgejo
-    state: started
-  when: forgejo_service.status.ActiveState == "active"
-
-- name: Display backup information
+- name: Display found backup files
   debug:
     msg: |
-      Forgejo backup completed successfully!
+      Found {{ existing_dumps.files | length }} dump files
+      Created full backup archive: /tmp/forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz
 
-      Backup files saved to: {{ backup_dest_dir }}/
-      - Forgejo dump: {{ dump_file.files[0].path | basename }}
-      - Additional data: forgejo-additional-{{ ansible_date_time.date }}.tar.gz
+- name: Fetch dump file to ops server (if exists)
+  synchronize:
+    mode: pull
+    src: "{{ (existing_dumps.files | sort(attribute='mtime', reverse=true) | first).path }}"
+    dest: /opt/forgejo-backup/
+  delegate_to: asxpio-ops
+  when: existing_dumps.files | length > 0
+  ignore_errors: true
 
-      Next steps:
-      1. Verify backup integrity
-      2. Run restore playbook: ansible-playbook playbooks/restore-forgejo.yaml
+- name: Fetch full backup archive to ops server
+  synchronize:
+    mode: pull
+    src: /tmp/forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz
+    dest: /opt/forgejo-backup/
+  delegate_to: asxpio-ops
+  become: true
+
+- name: Display backup location
+  debug:
+    msg: |
+      Backup files transferred to asxpio-ops:/opt/forgejo-backup/
+
+      Files:
+      - forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz (complete backup)
+      {% if existing_dumps.files | length > 0 %}
+      - {{ (existing_dumps.files | sort(attribute='mtime', reverse=true) | first).path | basename }} (dump file)
+      {% endif %}
+
+      Next step: Run restore playbook
+      ansible-playbook playbooks/restore-forgejo.yaml

ansible/roles/forgejo_restore/tasks/main.yaml

@@ -1,89 +1,112 @@
 ---
+- name: Find backup files on server
+  find:
+    paths: "{{ backup_dest_dir }}"
+    patterns: "forgejo-full-backup-*.tar.gz"
+  register: server_backup_files
+
+- name: Fail if no backup files found
+  fail:
+    msg: "No backup files found in {{ backup_dest_dir }}. Please run backup-forgejo.yaml first."
+  when: server_backup_files.files | length == 0
+
+- name: Select most recent backup file
+  set_fact:
+    backup_file: "{{ (server_backup_files.files | sort(attribute='mtime', reverse=true) | first).path }}"
+
+- name: Display backup file to be restored
+  debug:
+    msg: "Restoring from: {{ backup_file }}"
+
 - name: Stop Forgejo container
   community.docker.docker_compose_v2:
     project_src: /opt/forgejo
     state: stopped
   ignore_errors: true
 
-- name: Find backup dump file locally
-  find:
-    paths: "{{ backup_dest_dir }}"
-    patterns: "gitea-dump-*.zip"
-  delegate_to: localhost
-  become: false
-  register: local_dump_file
+- name: Backup current Forgejo data (if exists)
+  archive:
+    path: /opt/forgejo/data
+    dest: "/opt/forgejo/data-backup-{{ ansible_date_time.epoch }}.tar.gz"
+    format: gz
+  ignore_errors: true
 
-- name: Find additional backup file locally
-  find:
-    paths: "{{ backup_dest_dir }}"
-    patterns: "forgejo-additional-*.tar.gz"
-  delegate_to: localhost
-  become: false
-  register: local_additional_file
-
-- name: Fail if no backup files found
-  fail:
-    msg: "No backup files found in {{ backup_dest_dir }}. Please run backup-forgejo.yaml first."
-  when: local_dump_file.files | length == 0
-
-- name: Copy dump backup to server
-  copy:
-    src: "{{ local_dump_file.files[0].path }}"
-    dest: /tmp/forgejo-dump.zip
-    mode: '0644'
-
-- name: Copy additional backup to server
-  copy:
-    src: "{{ local_additional_file.files[0].path }}"
-    dest: /tmp/forgejo-additional.tar.gz
-    mode: '0644'
-  when: local_additional_file.files | length > 0
-
-- name: Extract Forgejo dump
-  unarchive:
-    src: /tmp/forgejo-dump.zip
-    dest: /opt/forgejo/
-    remote_src: true
-    creates: /opt/forgejo/app.ini
-
-- name: Extract additional backup
-  unarchive:
-    src: /tmp/forgejo-additional.tar.gz
-    dest: /tmp/forgejo-restore/
-    remote_src: true
-  when: local_additional_file.files | length > 0
-
-- name: Ensure data directory exists
+- name: Remove old data directory
   file:
     path: /opt/forgejo/data
+    state: absent
+
+- name: Create fresh data directory structure
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: 1000
+    group: 1000
+    mode: "{{ item.mode | default('0755') }}"
+  loop:
+    - { path: '/opt/forgejo/data' }
+    - { path: '/opt/forgejo/data/gitea' }
+    - { path: '/opt/forgejo/data/gitea/data' }
+    - { path: '/opt/forgejo/data/git' }
+    - { path: '/opt/forgejo/data/git/repositories' }
+    - { path: '/opt/forgejo/data/git/.ssh', mode: '0700' }
+
+- name: Create temporary restore directory
+  file:
+    path: /tmp/forgejo-restore
     state: directory
     mode: '0755'
 
-- name: Copy database from dump to data directory
-  copy:
-    src: /opt/forgejo/data/gitea.db
-    dest: /opt/forgejo/data/gitea.db
+- name: Extract backup to temporary location
+  unarchive:
+    src: "{{ backup_file }}"
+    dest: /tmp/forgejo-restore/
     remote_src: true
-    force: true
-  ignore_errors: true
 
-- name: Copy repository data from additional backup
+- name: Copy database from backup
+  copy:
+    src: /tmp/forgejo-restore/gitea.db
+    dest: /opt/forgejo/data/gitea/gitea.db
+    remote_src: true
+    owner: 1000
+    group: 1000
+    mode: '0644'
+
+- name: Copy avatars from backup
   synchronize:
-    src: /tmp/forgejo-restore/srv/gitea/data/
-    dest: /opt/forgejo/data/
+    src: /tmp/forgejo-restore/data/avatars/
+    dest: /opt/forgejo/data/gitea/data/avatars/
   delegate_to: "{{ inventory_hostname }}"
-  when: local_additional_file.files | length > 0
   ignore_errors: true
 
-- name: Copy SSH keys from additional backup
+- name: Copy repo-avatars from backup
   synchronize:
-    src: /tmp/forgejo-restore/var/lib/forgejo/.ssh/
-    dest: /opt/forgejo/data/.ssh/
+    src: /tmp/forgejo-restore/data/repo-avatars/
+    dest: /opt/forgejo/data/gitea/data/repo-avatars/
   delegate_to: "{{ inventory_hostname }}"
-  when: local_additional_file.files | length > 0
   ignore_errors: true
 
-- name: Set correct ownership for Forgejo data
+- name: Copy repo-archive from backup
+  synchronize:
+    src: /tmp/forgejo-restore/data/repo-archive/
+    dest: /opt/forgejo/data/gitea/data/repo-archive/
+  delegate_to: "{{ inventory_hostname }}"
+  ignore_errors: true
+
+- name: Copy attachments from backup
+  synchronize:
+    src: /tmp/forgejo-restore/data/attachments/
+    dest: /opt/forgejo/data/gitea/data/attachments/
+  delegate_to: "{{ inventory_hostname }}"
+  ignore_errors: true
+
+- name: Copy repositories from backup
+  synchronize:
+    src: /tmp/forgejo-restore/data/gitea-repositories/
+    dest: /opt/forgejo/data/git/repositories/
+  delegate_to: "{{ inventory_hostname }}"
+
+- name: Set correct ownership for all Forgejo data
   file:
     path: /opt/forgejo/data
     owner: 1000
@@ -96,34 +119,37 @@
     state: present
 
 - name: Wait for Forgejo to be healthy
-  uri:
-    url: http://localhost:{{ forgejo_http_port }}/
-    status_code: 200
-  retries: 15
-  delay: 5
-  register: result
-  until: result.status == 200
+  wait_for:
+    host: localhost
+    port: "{{ forgejo_http_port }}"
+    timeout: 60
+    delay: 5
 
 - name: Clean up temporary files
   file:
-    path: "{{ item }}"
+    path: /tmp/forgejo-restore
     state: absent
-  loop:
-    - /tmp/forgejo-dump.zip
-    - /tmp/forgejo-additional.tar.gz
-    - /tmp/forgejo-restore
+
+- name: Verify users in database
+  shell: docker exec -u git forgejo forgejo admin user list
+  register: user_list
+  ignore_errors: true
 
 - name: Display restore information
   debug:
     msg: |
       Forgejo restore completed successfully!
 
+      Users:
+      {{ user_list.stdout }}
+
       Forgejo is now running at:
       - Internal: http://localhost:{{ forgejo_http_port }}
       - External: https://{{ forgejo_domain }}
 
       Next steps:
-      1. Test git operations: git clone https://{{ forgejo_domain }}/user/repo.git
-      2. Test SSH access: git clone git@{{ forgejo_domain }}:user/repo.git
-      3. Test web interface at https://{{ forgejo_domain }}
-      4. Register runners with the new instance
+      1. Test web interface: https://{{ forgejo_domain }}
+      2. Verify avatars and images are visible
+      3. Test login with your credentials
+      4. Test git clone: git clone https://{{ forgejo_domain }}/user/repo.git
+      5. Check logs: docker logs forgejo

ansible/roles/forgejo_runner/tasks/main.yaml

@@ -29,17 +29,58 @@
     state: present
   register: runner_output
 
-- name: Display runner registration instructions
+- name: Check if runner is already registered
+  ansible.builtin.stat:
+    path: /opt/forgejo-runner/data/.runner
+  register: runner_registered
+
+- name: Set permissions on data directory
+  ansible.builtin.file:
+    path: /opt/forgejo-runner/data
+    state: directory
+    mode: '0777'
+    recurse: true
+  when: not runner_registered.stat.exists
+
+- name: Stop runner for registration
+  community.docker.docker_compose_v2:
+    project_src: /opt/forgejo-runner
+    state: stopped
+  when: not runner_registered.stat.exists
+
+- name: Register runner with Forgejo
+  ansible.builtin.command:
+    cmd: >
+      docker run --rm
+      -v /opt/forgejo-runner/data:/data
+      -v /opt/forgejo-runner/config/config.yaml:/config.yaml
+      code.forgejo.org/forgejo/runner:{{ forgejo_runner_version }}
+      forgejo-runner register
+      --no-interactive
+      --token {{ forgejo_runner_token }}
+      --name runner-{{ inventory_hostname }}
+      --instance http://10.10.0.10:{{ forgejo_http_port }}
+      --labels ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:22-bookworm,debian-latest:docker://debian:bookworm,debian-12:docker://debian:bookworm,debian-11:docker://debian:bullseye,arch-latest:docker://archlinux:latest,nginx-latest:docker://nginx:latest,nginx-alpine:docker://nginx:alpine,node-20:docker://node:20,node-22:docker://node:22,python-3.11:docker://python:3.11,python-3.12:docker://python:3.12,alpine-latest:docker://alpine:latest
+  when: not runner_registered.stat.exists
+  register: registration_result
+
+- name: Start runner after registration
+  community.docker.docker_compose_v2:
+    project_src: /opt/forgejo-runner
+    state: present
+  when: not runner_registered.stat.exists
+
+- name: Display runner status
   debug:
     msg: |
       Forgejo Runner deployed successfully!
 
-      To register the runner with Forgejo:
-      1. Log in to Forgejo at https://{{ forgejo_domain }}
-      2. Go to Site Administration > Actions > Runners
-      3. Generate a registration token
-      4. Run: docker exec -it forgejo-runner forgejo-runner register --no-interactive \
-           --token YOUR_TOKEN \
-           --name runner-{{ inventory_hostname }} \
-           --instance http://10.10.0.10:{{ forgejo_http_port }}
-      5. Restart the runner: cd /opt/forgejo-runner && docker compose restart
+      Runner name: runner-{{ inventory_hostname }}
+      Forgejo instance: http://10.10.0.10:{{ forgejo_http_port }}
+      {% if runner_registered.stat.exists %}
+      Status: Already registered (using existing registration)
+      {% else %}
+      Status: Newly registered
+      {% endif %}
+
+      Check runner status: docker logs forgejo-runner

ansible/roles/forgejo_runner/templates/docker-compose.yml.j2

@@ -3,6 +3,8 @@ services:
     image: code.forgejo.org/forgejo/runner:{{ forgejo_runner_version }}
     container_name: forgejo-runner
     restart: unless-stopped
+    user: "0:0"  # Run as root to access docker socket
+    command: forgejo-runner daemon
     volumes:
       - ./data:/data
       - ./config/config.yaml:/config.yaml

ansible/roles/ssh_hardening/defaults/main.yaml

@@ -17,7 +17,7 @@ ssh_shell_users:
   - root
 
 # Git user for Forgejo (allowed from anywhere when ssh_allow_public_git is true)
-ssh_git_user: forgejo
+ssh_git_user: git
 
 # SSH hardening settings
 ssh_max_auth_tries: 3

ansible/roles/traefik/tasks/main.yaml

@@ -44,10 +44,8 @@
   register: traefik_output
 
 - name: Wait for Traefik to be healthy
-  uri:
-    url: http://localhost:8080/ping
-    status_code: 200
-  retries: 10
-  delay: 3
-  register: result
-  until: result.status == 200
+  wait_for:
+    host: "{{ wireguard_ipv4 }}"
+    port: 8080
+    timeout: 30
+    delay: 3

ansible/roles/traefik/templates/docker-compose.yml.j2

@@ -10,7 +10,7 @@ services:
     ports:
       - "80:80"
       - "443:443"
-      - "10.20.0.10:8080:8080"  # Dashboard only on WireGuard IP
+      - "{{ wireguard_ipv4 }}:8080:8080"  # Dashboard only on WireGuard IP
     environment:
       - TZ=UTC
     volumes: