update README.md

README.md

@@ -1,43 +1,50 @@
 ## Prerequisites
+
 - **Bash**
 - **Docker**
 - **Docker compose**
 - **Ports:**
-  - **Caddy**
-    - 80/tcp (HTTP)
-    - 443/tcp (HTTPS)
-  - **Stalwart**
-    - 25/tcp (SMTP)
-    - 110/tcp (POP3)
-    - 995/tcp (POP3S)
-    - 143/tcp (IMAP)
-    - 993/tcp (IMAPS)
-    - 465/tcp (SMTPS)
-    - 587/tcp (SUBMISSION)
-    - 4190/tcp (ManageSieve)
-  - **Croc**
-    - 9009-9013/tcp (relay)
-  - **SFTPGo**
-    - 2022/tcp (SFTP)
-  - **WireGuard Easy**
-    - 51820/udp (WireGuard)
+    - **Caddy**
+        - 80/tcp (HTTP)
+        - 443/tcp (HTTPS)
+    - **Stalwart**
+        - 25/tcp (SMTP)
+        - 110/tcp (POP3)
+        - 995/tcp (POP3S)
+        - 143/tcp (IMAP)
+        - 993/tcp (IMAPS)
+        - 465/tcp (SMTPS)
+        - 587/tcp (SUBMISSION)
+        - 4190/tcp (ManageSieve)
+    - **Croc**
+        - 9009-9013/tcp (relay)
+    - **SFTPGo**
+        - 2022/tcp (SFTP)
+    - **WireGuard Easy**
+        - 51820/udp (WireGuard)
 
 ## Getting Started
 
 Follow these steps to set up and start the services:
+
 ### 1. Grant Execute Permissions
+
 Ensure the `main.sh` script has the necessary permissions:
+
 ```sh
 chmod +x main.sh
 ```
 
 ### 2. Generate Environment Files
+
 Create `.env` configuration files with the following command:
+
 ```sh
 ./main.sh generate-env
 ```
 
 ### 3. Configure Environment Variables
+
 Edit the generated `.env` files to fill in the required fields:
 
 - `./gitea/.env`
@@ -50,16 +57,67 @@ Edit the generated `.env` files to fill in the required fields:
 - `./croc/.env`
 - `./stalwart/.env`
 - `./caddy/.env`
+- `./crowdsec/.env`
 - `./caddy/Caddyfile.private`
 
-### 4. Start Services
+### 4. Bouncer Keys (CrowdSec)
+
+Generate two keys and write them into the matching `.env` files:
+
+```sh
+CADDY_KEY=$(openssl rand -hex 32)
+FW_KEY=$(openssl rand -hex 32)
+
+# crowdsec/.env
+sed -i "s|^CROWDSEC_BOUNCER_KEY_CADDY=.*|CROWDSEC_BOUNCER_KEY_CADDY=$CADDY_KEY|" ./crowdsec/.env
+sed -i "s|^CROWDSEC_BOUNCER_KEY_FW=.*|CROWDSEC_BOUNCER_KEY_FW=$FW_KEY|" ./crowdsec/.env
+
+# caddy/.env (same value as CADDY key above)
+sed -i "s|^CROWDSEC_API_KEY=.*|CROWDSEC_API_KEY=$CADDY_KEY|" ./caddy/.env
+```
+
+(Optional) get a Console enroll key from https://app.crowdsec.net and put it in `CROWDSEC_ENROLL_KEY`.
+
+### 5. Start Services
+
 Launch all services with the following command:
+
 ```sh
 ./main.sh start
 ```
+
+### 6. Host Firewall Bouncer (CrowdSec, nftables)
+
+The Caddy bouncer protects HTTP services. Stalwart's mail ports (25/465/587/143/993/110/995/4190) bypass Caddy, so install a firewall bouncer on the host. CrowdSec packages live on PackageCloud, not in the default apt repos, so add the repo first:
+
+```sh
+curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec/script.deb.sh | sudo bash
+sudo apt install crowdsec-firewall-bouncer-nftables
+```
+
+Do NOT use `install.crowdsec.net` (that installs the engine too, which we already run in Docker).
+
+Patch the default config (the package writes `api_url: http://127.0.0.1:8080/` but our LAPI is on 18080):
+
+```sh
+FW_KEY=$(grep '^CROWDSEC_BOUNCER_KEY_FW=' ./crowdsec/.env | cut -d= -f2)
+sudo sed -i "s|^api_url:.*|api_url: http://127.0.0.1:18080/|" /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
+sudo sed -i "s|^api_key:.*|api_key: $FW_KEY|" /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
+sudo systemctl enable --now crowdsec-firewall-bouncer
+sudo systemctl status crowdsec-firewall-bouncer --no-pager
+```
+
+Verify:
+
+```sh
+docker exec crowdsec cscli bouncers list      # 'firewall' should appear with a non-empty IP and recent 'Last API pull'
+sudo nft list ruleset | grep crowdsec         # kernel-level rules in place
+```
+
 ## Stopping Services
 
 To stop all running services, use:
+
 ```sh
 ./main.sh stop
 ```

caddy/.env.example

@@ -44,3 +44,7 @@ GOPKG_PROXY_DOMAIN=
 
 ############# Ech0 #############
 ECH0_DOMAIN=
+
+############# CrowdSec #############
+# Same value as CROWDSEC_BOUNCER_KEY_CADDY in crowdsec/.env
+CROWDSEC_API_KEY=

caddy/Caddyfile

@@ -3,11 +3,32 @@
     metrics {
         per_host
     }
-#    log {
-#        output file /var/log/caddy/access.log
-#        format json
-#        level DEBUG
-#    }
+
+    crowdsec {
+        api_url        http://crowdsec:8080
+        api_key        {env.CROWDSEC_API_KEY}
+        ticker_interval 15s
+        appsec_url     http://crowdsec:7422
+        appsec_fail_open
+        appsec_timeout 1s
+    }
+
+    servers {
+        trusted_proxies static private_ranges
+        client_ip_headers X-Forwarded-For
+    }
+}
+
+(access-log) {
+    log {
+        output stdout
+        format json
+    }
+}
+
+(security) {
+    crowdsec
+    appsec
 }
 
 :2019 {
@@ -16,158 +37,202 @@
 
 ############## grafana ##############
 {$GRAFANA_DOMAIN} {
+    import access-log
     request_body {
         max_size 2048MB
     }
-
-    reverse_proxy http://grafana:3000
+    route {
+        import security
+        reverse_proxy http://grafana:3000
+    }
 }
 
 ############## gitea ##############
 {$GITEA_DOMAIN} {
+    import access-log
     request_body {
         max_size 512MB
     }
-
-    reverse_proxy http://gitea:3000
+    route {
+        import security
+        reverse_proxy http://gitea:3000
+    }
 }
 
 ############## slash ##############
 {$SLASH_DOMAIN} {
+    import access-log
     request_body {
         max_size 10MB
     }
-
-    reverse_proxy http://slash:5231
+    route {
+        import security
+        reverse_proxy http://slash:5231
+    }
 }
 
 ############## memos ##############
 {$MEMOS_DOMAIN} {
+    import access-log
     request_body {
         max_size 1024MB
     }
-
-    reverse_proxy http://memos:5230
+    route {
+        import security
+        reverse_proxy http://memos:5230
+    }
 }
 
 ############## wg-easy ##############
 {$WG_EASY_DOMAIN} {
+    import access-log
     request_body {
         max_size 10MB
     }
-
-    reverse_proxy http://wg-easy:51821
+    route {
+        import security
+        reverse_proxy http://wg-easy:51821
+    }
 }
 
 ############## vaultwarden ##############
 {$VAULTWARDEN_DOMAIN} {
+    import access-log
     request_body {
         max_size 128MB
     }
-
-    reverse_proxy http://vaultwarden
+    route {
+        import security
+        reverse_proxy http://vaultwarden
+    }
 }
 
 ############## sftpgo ##############
 {$SFTPGO_DOMAIN} {
+    import access-log
     request_body {
         max_size 8120MB
     }
-
-    reverse_proxy http://sftpgo:8080
+    route {
+        import security
+        reverse_proxy http://sftpgo:8080
+    }
 }
 
 ############## glance ##############
 {$GLANCE_DOMAIN} {
+    import access-log
     request_body {
         max_size 64MB
     }
-
-    reverse_proxy http://glance:8080
+    route {
+        import security
+        reverse_proxy http://glance:8080
+    }
 }
 
 ############## ghost ##############
 {$GHOST_DOMAIN} {
+    import access-log
     request_body {
         max_size 124MB
     }
-
-    reverse_proxy http://ghost:2368 {
-        header_up X-Forwarded-Proto {http.request.scheme}
-        header_up Host {http.request.host}
+    route {
+        import security
+        reverse_proxy http://ghost:2368 {
+            header_up X-Forwarded-Proto {http.request.scheme}
+            header_up Host {http.request.host}
+        }
     }
 }
 
 ############## immich ##############
 {$IMMICH_DOMAIN} {
+    import access-log
     request_body {
         max_size 1024MB
     }
-
-    reverse_proxy http://immich_server:2283 {
-        header_up X-Forwarded-Proto {http.request.scheme}
-        header_up Host {http.request.host}
+    route {
+        import security
+        reverse_proxy http://immich_server:2283 {
+            header_up X-Forwarded-Proto {http.request.scheme}
+            header_up Host {http.request.host}
+        }
     }
 }
 
 ############## uptime-kuma ##############
 {$UPTIME_KUMA_DOMAIN} {
+    import access-log
     request_body {
         max_size 1024MB
     }
-
-    reverse_proxy http://uptime_kuma:3001 {
-        header_up X-Forwarded-Proto {http.request.scheme}
-        header_up Host {http.request.host}
+    route {
+        import security
+        reverse_proxy http://uptime_kuma:3001 {
+            header_up X-Forwarded-Proto {http.request.scheme}
+            header_up Host {http.request.host}
+        }
     }
 }
 
 ############## stalwart ##############
 {$STALWART_DOMAIN}, {$STALWART_AUTOCONFIG_DOMAIN}, {$STALWART_AUTODISCOVER_DOMAIN} {
+    import access-log
     request_body {
         max_size 4048MB
     }
-
-    reverse_proxy http://stalwart:8080 {
-        header_up X-Forwarded-Proto {http.request.scheme}
-        header_up X-Forwarded-For {http.request.remote_host}
-        header_up Host {http.request.host}
+    route {
+        import security
+        reverse_proxy http://stalwart:8080 {
+            header_up X-Forwarded-Proto {http.request.scheme}
+            header_up X-Forwarded-For {http.request.remote_host}
+            header_up Host {http.request.host}
+        }
     }
 }
 
 ############## textarea ##############
 {$TEXTAREA_DOMAIN} {
-    root * /volume/textarea
-    file_server {
-        browse off
+    import access-log
+    route {
+        import security
+        root * /volume/textarea
+        file_server {
+            browse off
+        }
     }
 }
 
 ############## gopkg-proxy ##############
 {$GOPKG_PROXY_DOMAIN} {
+    import access-log
     request_body {
         max_size 2MB
     }
-
-    reverse_proxy http://gopkg_proxy:8421
+    route {
+        import security
+        reverse_proxy http://gopkg_proxy:8421
+    }
 }
 
 ############## ech0 ##############
 {$ECH0_DOMAIN} {
+    import access-log
     header -Server
-
     request_body {
         max_size 5MB
     }
+    route {
+        import security
+        reverse_proxy http://ech0:8421 {
+            header_up -X-Forwarded-Host
+            header_up -X-Forwarded-Proto
+            header_up -Via
 
-    reverse_proxy http://ech0:8421 {
-        # wheader_up -X-Forwarded-For
-        header_up -X-Forwarded-Host
-        header_up -X-Forwarded-Proto
-        header_up -Via
-
-        transport http {
-            compression off
+            transport http {
+                compression off
+            }
         }
     }
 }

caddy/Dockerfile

@@ -0,0 +1,7 @@
+FROM caddy:2-builder-alpine AS builder
+RUN xcaddy build \
+    --with github.com/hslatman/caddy-crowdsec-bouncer/http \
+    --with github.com/hslatman/caddy-crowdsec-bouncer/appsec
+
+FROM caddy:2-alpine
+COPY --from=builder /usr/bin/caddy /usr/bin/caddy

caddy/docker-compose.yaml

@@ -6,11 +6,9 @@ networks:
 
 services:
   caddy:
-    image: caddy:2-alpine
+    build: .
     container_name: caddy
     restart: unless-stopped
-    labels:
-      - "com.centurylinklabs.watchtower.enable=true"
     networks:
       - caddy
     ports:

crowdsec/.env.example

@@ -0,0 +1,12 @@
+############# Bouncer keys #############
+# Generate with: openssl rand -hex 32
+# These are PRE-SEEDED into CrowdSec on first boot. Use the same value here
+# and in caddy/.env (CADDY) and the host firewall bouncer config (FW).
+CROWDSEC_BOUNCER_KEY_CADDY=
+CROWDSEC_BOUNCER_KEY_FW=
+
+############# Console enrollment #############
+# Enroll key from https://app.crowdsec.net (free).
+# Leave blank to run without Console (no community blocklist subscriptions).
+CROWDSEC_ENROLL_KEY=
+CROWDSEC_ENROLL_INSTANCE_NAME=

crowdsec/acquis.d/appsec.yaml

@@ -0,0 +1,7 @@
+appsec_configs:
+  - crowdsecurity/appsec-default
+labels:
+  type: appsec
+listen_addr: 0.0.0.0:7422
+source: appsec
+name: caddy-appsec

crowdsec/acquis.d/caddy.yaml

@@ -0,0 +1,7 @@
+source: docker
+container_name:
+  - caddy
+labels:
+  type: caddy
+follow_stdout: true
+follow_stderr: true

crowdsec/acquis.d/services.yaml

@@ -0,0 +1,23 @@
+source: docker
+container_name:
+  - gitea
+labels:
+  type: gitea
+follow_stdout: true
+follow_stderr: true
+---
+source: docker
+container_name:
+  - vaultwarden
+labels:
+  type: vaultwarden
+follow_stdout: true
+follow_stderr: true
+---
+source: docker
+container_name:
+  - stalwart
+labels:
+  type: stalwart
+follow_stdout: true
+follow_stderr: true

crowdsec/acquis.d/sshd.yaml

@@ -0,0 +1,5 @@
+source: journalctl
+journalctl_filter:
+  - "_SYSTEMD_UNIT=ssh.service"
+labels:
+  type: syslog

crowdsec/data/.gitignore

@@ -0,0 +1,3 @@
+/db/*
+/config/*
+!.gitkeep

crowdsec/docker-compose.yaml

@@ -0,0 +1,53 @@
+networks:
+  caddy:
+    name: caddy
+    driver: bridge
+    external: true
+
+services:
+  crowdsec:
+    image: crowdsecurity/crowdsec:latest-debian
+    container_name: crowdsec
+    restart: unless-stopped
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+    environment:
+      TZ: "Etc/UTC"
+      USE_WAL: "true"
+      COLLECTIONS: >-
+        crowdsecurity/linux
+        crowdsecurity/caddy
+        crowdsecurity/http-cve
+        crowdsecurity/sshd
+        crowdsecurity/whitelist-good-actors
+        crowdsecurity/appsec-virtual-patching
+        crowdsecurity/appsec-generic-rules
+        LePresidente/gitea
+        Dominic-Wagner/vaultwarden
+      BOUNCER_KEY_caddy: "${CROWDSEC_BOUNCER_KEY_CADDY}"
+      BOUNCER_KEY_firewall: "${CROWDSEC_BOUNCER_KEY_FW}"
+      ENROLL_KEY: "${CROWDSEC_ENROLL_KEY:-}"
+      ENROLL_INSTANCE_NAME: "${CROWDSEC_ENROLL_INSTANCE_NAME:-aykhans-prod}"
+    ports:
+      - "127.0.0.1:18080:8080"
+    networks:
+      - caddy
+    volumes:
+      - ./data/db:/var/lib/crowdsec/data
+      - ./data/config:/etc/crowdsec
+      - ./acquis.d:/etc/crowdsec/acquis.d:ro
+      - ./parsers/s00-raw/stalwart-logs.yaml:/etc/crowdsec/parsers/s00-raw/stalwart-logs.yaml:ro
+      - ./parsers/s01-parse/stalwart-logs-extended.yaml:/etc/crowdsec/parsers/s01-parse/stalwart-logs-extended.yaml:ro
+      - ./parsers/s02-enrich/whitelist-trusted.yaml:/etc/crowdsec/parsers/s02-enrich/whitelist-trusted.yaml:ro
+      - ./scenarios/stalwart-smtp-bruteforce.yaml:/etc/crowdsec/scenarios/stalwart-smtp-bruteforce.yaml:ro
+      - ./scenarios/stalwart-auth-bruteforce.yaml:/etc/crowdsec/scenarios/stalwart-auth-bruteforce.yaml:ro
+      - ./scenarios/http-admin-interface-probing.yaml:/etc/crowdsec/scenarios/http-admin-interface-probing.yaml:ro
+      - /var/log/journal:/var/log/journal:ro
+      - /run/log/journal:/run/log/journal:ro
+      - /etc/machine-id:/etc/machine-id:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "3"

crowdsec/parsers/s00-raw/stalwart-logs.yaml

@@ -0,0 +1,10 @@
+name: stalwart/parse-logs
+description: Raw parser for Stalwart logs
+stage: s00-raw
+onsuccess: next_stage
+filter: "evt.Parsed.program == 'stalwart' || evt.Line.Labels.type == 'stalwart'"
+
+nodes:
+  - grok:
+      apply_on: Line.Raw
+      pattern: '^%{TIMESTAMP_ISO8601:timestamp}\s+%{WORD:log_level}\s+%{DATA:message_text}\s*\(%{DATA:event_type}\)\s*(%{GREEDYDATA:kvpairs})?$'

crowdsec/parsers/s01-parse/stalwart-logs-extended.yaml

@@ -0,0 +1,15 @@
+name: stalwart/parse-extended
+description: Parse Stalwart logs including key fields without kv parser
+stage: s01-parse
+onsuccess: next_stage
+filter: "evt.Parsed.program == 'stalwart' || evt.Line.Labels.type == 'stalwart'"
+
+nodes:
+  - grok:
+      apply_on: Line.Raw
+      pattern: '^%{TIMESTAMP_ISO8601:timestamp}\s+%{WORD:log_level}\s+%{DATA:message_text}\s+\(%{DATA:event_type}\)\s*(?:listenerId\s*=\s*"%{DATA:listenerId}",\s*)?(?:localPort\s*=\s*%{INT:localPort},\s*)?(?:remoteIp\s*=\s*%{IP:remoteIp},\s*)?(?:remotePort\s*=\s*%{INT:remotePort},\s*)?(?:reason\s*=\s*"%{DATA:reason}")?.*$'
+    statics:
+      - meta: source_ip
+        expression: evt.Parsed.remoteIp
+      - meta: service
+        value: stalwart

crowdsec/parsers/s02-enrich/whitelist-trusted.yaml

@@ -0,0 +1,11 @@
+name: aykhans/whitelist-trusted
+description: Trusted operator IPs and Docker private network ranges
+whitelist:
+  reason: "operator + internal docker networks"
+  ip:
+    - "127.0.0.1"
+    - "::1"
+  cidr:
+    - "10.0.0.0/8"
+    - "172.16.0.0/12"
+    - "192.168.0.0/16"

crowdsec/scenarios/http-admin-interface-probing.yaml

@@ -0,0 +1,26 @@
+type: leaky
+name: crowdsecurity/http-admin-interface-probing
+description: "Detect generic HTTP admin interface probing"
+filter: |
+  evt.Meta.service == 'http' and
+  evt.Meta.log_type in ['http_access-log', 'http_error-log'] and
+  evt.Meta.http_status in ['404', '403'] and
+  any(File("admin_interfaces.txt"), { Lower(evt.Meta.http_path) contains #})
+groupby: evt.Meta.source_ip
+distinct: "evt.Meta.http_path"
+data:
+  - source_url: https://hub-data.crowdsec.net/web/admin_interfaces.txt
+    dest_file: admin_interfaces.txt
+    type: string
+capacity: 5
+leakspeed: "10s"
+blackhole: 1m
+labels:
+  confidence: 3
+  spoofable: 0
+  classification:
+    - attack.T1595
+  behavior: "http:scan"
+  label: "HTTP Admin Interface Probing"
+  service: http
+  remediation: true

crowdsec/scenarios/stalwart-auth-bruteforce.yaml

@@ -0,0 +1,13 @@
+type: leaky
+name: aykhans/stalwart-auth-bruteforce
+description: Detect SMTP/IMAP/POP3 authentication brute force on Stalwart
+filter: |
+  evt.Parsed.event_type == "auth.failed"
+groupby: evt.Meta.source_ip
+capacity: 3
+leakspeed: 10m
+blackhole: 1h
+labels:
+  type: bruteforce
+  service: stalwart
+  remediation: true

crowdsec/scenarios/stalwart-smtp-bruteforce.yaml

@@ -0,0 +1,17 @@
+type: leaky
+name: stalwart/smtp-bruteforce
+description: Detect SMTP bruteforce and scanners on Stalwart logs
+filter: |
+  evt.Parsed.event_type in [
+    "smtp.invalid-ehlo",
+    "smtp.auth-not-allowed",
+    "smtp.auth-mechanism-not-supported"
+  ]
+groupby: evt.Meta.source_ip
+capacity: 5
+leakspeed: 10m
+blackhole: 1h
+labels:
+  type: attack
+  service: smtp
+  remediation: true

main.sh

@@ -64,6 +64,7 @@ generate_env_files() {
     cp --update=none ./uptime_kuma/.env.example ./uptime_kuma/.env
     cp --update=none ./croc/.env.example ./croc/.env
     cp --update=none ./stalwart/.env.example ./stalwart/.env
+    cp --update=none ./crowdsec/.env.example ./crowdsec/.env
     cp --update=none ./caddy/Caddyfile.private.example ./caddy/Caddyfile.private
     print_success ".env files generated."
 }
@@ -190,8 +191,17 @@ start_services() {
         exit 1
     fi
 
+    echo "Starting crowdsec..."
+    $DOCKER_COMPOSE_COMMAND -f ./crowdsec/docker-compose.yaml up --pull always -d
+    if [ $? -eq 0 ]; then
+        print_success "Crowdsec started successfully."
+    else
+        print_error "failed to start Crowdsec!"
+        exit 1
+    fi
+
     echo "Starting caddy..."
-    $DOCKER_COMPOSE_COMMAND -f ./caddy/docker-compose.yaml up --pull always -d
+    $DOCKER_COMPOSE_COMMAND -f ./caddy/docker-compose.yaml up --pull always --build -d
     if [ $? -eq 0 ]; then
         print_success "Caddy started successfully."
     else
@@ -368,6 +378,15 @@ stop_services() {
         exit 1
     fi
 
+    echo "Stopping crowdsec..."
+    $DOCKER_COMPOSE_COMMAND -f ./crowdsec/docker-compose.yaml down
+    if [ $? -eq 0 ]; then
+        print_success "Crowdsec stopped successfully."
+    else
+        print_error "failed to stop Crowdsec!"
+        exit 1
+    fi
+
     echo "Stopping watchtower..."
     $DOCKER_COMPOSE_COMMAND -f ./watchtower/docker-compose.yaml down
     if [ $? -eq 0 ]; then

vaultwarden/docker-compose.yaml

@@ -6,12 +6,13 @@ networks:
 
 services:
   server:
-    image: vaultwarden/server:1.35.7
+    image: vaultwarden/server:1.36.0
     container_name: vaultwarden
     restart: unless-stopped
     environment:
       - DOMAIN=${DOMAIN}
       - ADMIN_TOKEN=${ADMIN_TOKEN}
+      - LOG_TIMESTAMP_FORMAT=%Y-%m-%d %H:%M:%S.%3f%:z
     networks:
       - caddy
     volumes: