Troubleshooting

Troubleshooting Guide

Container Issues

Container won’t start

# Check container status
systemctl status myapp.service
 
# View logs
journalctl -u myapp.service -f
 
# Validate quadlet unit file
systemd-analyze verify /etc/containers/systemd/myapp.container
 
# Check if image exists
podman image inspect docker.io/org/image:latest
 
# Pull image manually
podman pull docker.io/org/image:latest

Port conflict

# Check what's using a port
ss -tlnp | grep :PORT
 
# Common conflicts:
# Port 53: dnsmasq vs AdGuard Home
# Solution: dnsmasq binds to lo only (interface=lo in dnsmasq.conf)

Permission denied on volumes

# Check SELinux context (if enforcing)
ls -Z /srv/cache1/container_data/myapp/
 
# Add :Z label (only if SELinux is enforcing)
Volume=/srv/cache1/container_data/myapp:/app/data:Z
 
# Check ownership
ls -la /srv/cache1/container_data/myapp/
chown -R 1000:1000 /srv/cache1/container_data/myapp/

DNS Issues

AdGuard Home not responding

# Check if container is running
systemctl status adguardhome.service
 
# Test DNS resolution
dig @127.0.0.1 example.com
 
# Check if port 53 is bound
ss -tlnp | grep :53
 
# Restart AdGuard
systemctl restart adguardhome.service

dnsmasq conflict with AdGuard

# Verify dnsmasq binds to loopback only
cat /etc/dnsmasq.conf | grep -E 'interface|bind-interfaces'
 
# Should show:
# interface=lo
# bind-interfaces
 
# If not, edit and restart
systemctl restart dnsmasq

nginx Issues

Configuration error

# Test configuration
nginx -t
 
# View error log
tail -f /var/log/nginx/error.log
 
# Reload configuration
systemctl reload nginx

Zabbix UI not accessible on port 9080

# Check nginx is running
systemctl status nginx
 
# Check zabbix.conf exists
ls /etc/nginx/conf.d/zabbix.conf
 
# Check PHP-FPM socket
ls -la /run/php-fpm/zabbix.sock
 
# Restart nginx and php-fpm
systemctl restart nginx php-fpm

Cloudflare Tunnel Issues

Tunnel not connecting

# Check service status
systemctl status cloudflared
 
# View logs for errors
journalctl -u cloudflared --since "10 minutes ago"
 
# Check network connectivity
curl -s https://www.cloudflare.com/cdn-cgi/trace
 
# Restart tunnel
systemctl restart cloudflared

Auto-update not working

# Run update manually
systemctl start cloudflared-update.service
 
# Check update logs
journalctl -u cloudflared-update.service -f

Monitoring Issues

Zabbix server not starting

# Check PostgreSQL
systemctl status postgresql
 
# Check Zabbix logs
tail -f /var/log/zabbix/zabbix_server.log
 
# Verify database connection
sudo -u postgres psql -c "\l"
 
# Restart in order
systemctl restart postgresql
systemctl restart zabbix-server
systemctl restart zabbix-agent

Prometheus not scraping metrics

# Check Prometheus config
cat /srv/cache1/container_data/prometheus/prometheus.yml
 
# Check target status (from Prometheus UI)
# http://localhost:9090/targets
 
# Verify exporters are running
podman ps | grep -E 'exporter|prometheus'
 
# Check network connectivity between containers
podman exec prometheus ping node-exporter

Grafana empty dashboards

# Check Prometheus data source configuration
# http://localhost:3000/datasources
 
# Verify Prometheus URL is reachable from Grafana
podman exec grafana curl -s http://prometheus:9090/metrics
 
# Check Grafana logs
podman logs grafana

Storage Issues

Disk space warnings

# Check disk usage
df -h /srv/cache1 /srv/disk1
 
# Find large files
du -sh /srv/cache1/* | sort -rh | head -10
 
# Clean up container images
podman system prune -a
 
# Clean up old logs
journalctl --vacuum-size=500M

NFS mount not accessible

# Check NFS exports
exportfs -v
 
# Check NFS service
systemctl status nfs-server
 
# Restart NFS
systemctl restart nfs-server
 
# Verify from client
showmount -e server-ip

Firewall Issues

Service not accessible from LAN

# Check firewall rules
firewall-cmd --list-all
 
# Check if port is in public zone
firewall-cmd --list-ports
 
# Add port if missing
firewall-cmd --add-port=PORT/tcp --permanent
firewall-cmd --reload
 
# Verify
firewall-cmd --query-port=PORT/tcp

firewalld blocking container traffic

# Check rich rules
firewall-cmd --list-rich-rules
 
# Allow container subnet (if needed)
firewall-cmd --add-rich-rule='rule family=ipv4 source address=10.88.0.0/16 accept' --permanent
firewall-cmd --reload

Quick Diagnostics

# Full system overview
systemctl list-units --type=service --state=running | grep -iE 'podman|container|zabbix|nginx|cloud|postgres|smb|nfs|dnsmasq'
 
# Container resource usage
podman stats --no-stream
 
# Disk usage
df -h
 
# Memory usage
free -h
 
# Network connections
ss -tlnp
 
# Recent errors in journal
journalctl -p err --since "1 hour ago"
 
# Check all podman containers
podman ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"