Sending syslog to a remote server is a good idea because your logs will be safe in the event you have a hardware failure or security breach.
Here is a guide on sending the syslog from your production servers to a single local destination that will then forward them securely (using stunnel) to another remote syslog server for backup or viewing.
[Local Servers (A,B)]--->[Local Syslog Aggregator (C)]===Secure Tunnel===>[Remote Syslog Receiver (D)]
Step 1 - Install syslog-ng on all servers A,B,C,D
apt-get install syslog-ng
Step 2 - Install stunnel on servers C and D
apt-get install stunnel
Step 3 - Configure servers A and B to send their syslog to C
#nano /etc/syslog-ng/syslog-ng.conf
...
#At the end of the file add:
#===========================
# Send syslog to aggregator
#===========================
destination loghost { tcp("[IP ADDRESS OF C]" port(5140)); };
log { source(s_src); destination(loghost); };
...
#At the end of the file add:
#===========================
# Send syslog to aggregator
#===========================
destination loghost { tcp("[IP ADDRESS OF C]" port(5140)); };
log { source(s_src); destination(loghost); };
Step 4 - Configure server C to accept incoming syslog and send it to stunnel for forwarding
#nano /etc/syslog-ng/syslog-ng.conf
...
#At the end of the file add:
#====================================
# Receive syslog and send to stunnel
#====================================
# Create source for incoming tcp logs from local servers
source incoming_src { tcp(ip("[IP ADDRESS OF C]") port(5140) keep-alive(yes) max-connections(16)); };
# Create destination for stunnel from localhost to remote site
destination dst_remote_via_stunnel { tcp("127.0.0.1" port(5141)); };
# Send incoming logs to remote
log { source(incoming_src); destination(dst_remote_via_stunnel); };
# Send own logs to remote (optional)
log { source(s_src); destination(dst_remote_via_stunnel); };
...
#At the end of the file add:
#====================================
# Receive syslog and send to stunnel
#====================================
# Create source for incoming tcp logs from local servers
source incoming_src { tcp(ip("[IP ADDRESS OF C]") port(5140) keep-alive(yes) max-connections(16)); };
# Create destination for stunnel from localhost to remote site
destination dst_remote_via_stunnel { tcp("127.0.0.1" port(5141)); };
# Send incoming logs to remote
log { source(incoming_src); destination(dst_remote_via_stunnel); };
# Send own logs to remote (optional)
log { source(s_src); destination(dst_remote_via_stunnel); };
Step 5 - Configure stunnel client on server C
#nano /etc/stunnel/stunnel.conf
; Protocol version (all, SSLv2, SSLv3, TLSv1)
sslVersion = SSLv3
; Some security enhancements for UNIX systems - comment them out on Win32
chroot = /var/lib/stunnel4/
setuid = stunnel4
setgid = stunnel4
; PID is created inside the chroot jail
pid = /stunnel4.pid
; Some performance tunings
socket = l:TCP_NODELAY=1
socket = r:TCP_NODELAY=1
; Some debugging stuff useful for troubleshooting
debug = 7
output = /var/log/stunnel4/stunnel.log
; Use it for client mode
client = yes
[syslog-ng]
accept = 127.0.0.1:5141
connect = [IP ADDRESS OF D]:5140
; Protocol version (all, SSLv2, SSLv3, TLSv1)
sslVersion = SSLv3
; Some security enhancements for UNIX systems - comment them out on Win32
chroot = /var/lib/stunnel4/
setuid = stunnel4
setgid = stunnel4
; PID is created inside the chroot jail
pid = /stunnel4.pid
; Some performance tunings
socket = l:TCP_NODELAY=1
socket = r:TCP_NODELAY=1
; Some debugging stuff useful for troubleshooting
debug = 7
output = /var/log/stunnel4/stunnel.log
; Use it for client mode
client = yes
[syslog-ng]
accept = 127.0.0.1:5141
connect = [IP ADDRESS OF D]:5140
Step 6 - Enable auto start of stunnel on C
#nano /etc/default/stunnel4
ENABLED=1
ENABLED=1
Step 7 - VERY IMPORTANT - Create a certificate and key on server D to encrypt the syslog within the tunnel to prevent eavesdropping of your syslogs. You do not need a certificate on the stunnel client (C) if you are restricting connections to the stunnel server (D) by source IP at the firewall.
openssl req -new -x509 -days 3650 -nodes -out /etc/stunnel/stunnel.pem -keyout /etc/stunnel/stunnel.pem
chmod 600 /etc/stunnel/stunnel.pem
chmod 600 /etc/stunnel/stunnel.pem
Step 8 - Configure stunnel on remote syslog receiver D
#nano /etc/stunnel/stunnel.conf
; Certificate/key is needed in server mode and optional in client mode
cert = /etc/stunnel/stunnel.pem
key = /etc/stunnel/stunnel.pem
; Protocol version (all, SSLv2, SSLv3, TLSv1)
sslVersion = SSLv3
; Some security enhancements for UNIX systems - comment them out on Win32
chroot = /var/lib/stunnel4/
setuid = stunnel4
setgid = stunnel4
; PID is created inside the chroot jail
pid = /stunnel4.pid
; Some performance tunings
socket = l:TCP_NODELAY=1
socket = r:TCP_NODELAY=1
; Some debugging stuff useful for troubleshooting
debug = 7
output = /var/log/stunnel4/stunnel.log
client = no
[syslog-ng]
accept = 0.0.0.0:5140
connect = 127.0.0.1:5141
; Certificate/key is needed in server mode and optional in client mode
cert = /etc/stunnel/stunnel.pem
key = /etc/stunnel/stunnel.pem
; Protocol version (all, SSLv2, SSLv3, TLSv1)
sslVersion = SSLv3
; Some security enhancements for UNIX systems - comment them out on Win32
chroot = /var/lib/stunnel4/
setuid = stunnel4
setgid = stunnel4
; PID is created inside the chroot jail
pid = /stunnel4.pid
; Some performance tunings
socket = l:TCP_NODELAY=1
socket = r:TCP_NODELAY=1
; Some debugging stuff useful for troubleshooting
debug = 7
output = /var/log/stunnel4/stunnel.log
client = no
[syslog-ng]
accept = 0.0.0.0:5140
connect = 127.0.0.1:5141
Step 9 - Configure stunnel on remote received D to accept stunnel as a log source
#nano /etc/syslog-ng/syslog-ng.conf
...
#At the end of the file add:
#Create a source for the incoming remote logs
source s_incoming_remote { tcp(ip("127.0.0.1") port(5141) keep-alive(yes)); };
#Create a destination that splits the incoming syslog into per-server per-day directories
destination d_remote_split { file("/var/log/PRODUCTION/$HOST/$YEAR.$MONTH.$DAY/messages"); };
#Connect the incoming remote syslog to the splitter destination
log { source(s_incoming_remote); destination(d_remote_split); };
...
#At the end of the file add:
#Create a source for the incoming remote logs
source s_incoming_remote { tcp(ip("127.0.0.1") port(5141) keep-alive(yes)); };
#Create a destination that splits the incoming syslog into per-server per-day directories
destination d_remote_split { file("/var/log/PRODUCTION/$HOST/$YEAR.$MONTH.$DAY/messages"); };
#Connect the incoming remote syslog to the splitter destination
log { source(s_incoming_remote); destination(d_remote_split); };
Step 10 - Enable auto start of stunnel on D
#nano /etc/default/stunnel4
ENABLED=1
ENABLED=1
Remember to open a firewall port on stunnel server (D) e.g. 5140 and only allow connections from server C.