# i=0; while true; do i=$(($i+1)); echo $i ==============================; netstat -natlp | grep ^tcp | sort -nk1 | awk '{ if($2 != 0) {print}}' ; sleep 1; done;
1 ==============================
2 ==============================
3 ==============================
4 ==============================
5 ==============================
tcp 100 0 10.0.3.167:22 198.21.8.23:53477 ESTABLISHED 99304/sshd: fordodone
6 ==============================
7 ==============================
8 ==============================
9 ==============================
tcp 43520 0 10.0.3.167:53877 10.0.9.55:3306 ESTABLISHED 119789/mysqldump
10 ==============================
11 ==============================
12 ==============================
13 ==============================
14 ==============================
15 ==============================
16 ==============================
tcp6 1 0 10.0.3.167:80 198.21.8.23:65114 CLOSE_WAIT 3880/apache2
17 ==============================
18 ==============================
Tag Archives: while
use dig to monitor DNS server for response
while true; do j=`dig < hostname > @W.X.Y.Z +short`; if [ "$j" != "1.2.3.4" ]; then echo nodig |mail 8185551212@vtext.com; break; fi ; sleep 1m; done;
This particular DNS server was only responding intermittently, so it needed some closer watching
wget monitor website download speed
# while true; do date | tr '\n' '-' | sed -e 's/-/ --- /'; wget http://testsite.com/fancy.pdf -O /dev/null 2>&1 | grep saved | awk -F"[()]" '{print $2}'; sleep 1s; done;
Thu Oct 30 15:18:26 PDT 2014 --- 1.25 MB/s
Thu Oct 30 15:18:28 PDT 2014 --- 1.20 MB/s
Thu Oct 30 15:18:29 PDT 2014 --- 958.95 KB/s
Thu Oct 30 15:18:31 PDT 2014 --- 1.36 MB/s
Thu Oct 30 15:18:32 PDT 2014 --- 873.98 KB/s
Thu Oct 30 15:18:33 PDT 2014 --- 1.38 MB/s
Thu Oct 30 15:18:35 PDT 2014 --- 261.90 KB/s
Thu Oct 30 15:18:37 PDT 2014 --- 1.38 MB/s
Thu Oct 30 15:18:38 PDT 2014 --- 360.14 KB/s
Thu Oct 30 15:18:40 PDT 2014 --- 1.37 MB/s
Thu Oct 30 15:18:42 PDT 2014 --- 427.06 KB/s
Thu Oct 30 15:18:44 PDT 2014 --- 1.37 MB/s
Thu Oct 30 15:18:45 PDT 2014 --- 397.54 KB/s
Vyatta monitor and log NAT translation
Logging to record NAT translations. This might be helpful for finding users using bit torrent (along with tshark), or for watching what IPs are connecting to what external services, and when.
while true; do d=`date +%Y%m%d%H%M`; show nat translations detail | grep -v Pre-NAT | paste - - | sort -nk1 >nats.$d.log; sleep 5m; done;
# tail -1 nats.201408261250.log
a.a.a.a:21845 z.z.z.z:443 b.b.b.b:21845 z.z.z.z:443 tcp: snat: a.a.a.a ==> b.b.b.b timeout: 42 use: 1
monitor NetApp SnapMirror transfer speed
You may want to monitor the speed of a current snapmirror to get an idea of how fast the transfer is going. The speed might change throughout the day due to load, or disk bottleneck, etc. I started with this one-liner:
i=0;j=0; while true; do j=$i; i=`ssh toaster01 "snapmirror status -l volname" | grep Progress | awk '{print $2}'| cut -d \( -f2`; if [ $j -eq 0 ]; then sleep 1m; continue; fi; echo -n "$i-$j = ";echo "scale=2;($i-$j)/(1024*1024)" | bc | tr '\n' ' '; echo "GB/min"; sleep 1m; done;
Which lead to this short script:
#!/bin/bash # FILE: netapp_sm_monitor.sh # AUTHOR: For Do Done <fordodone@fordodone.com> # DATE: 2014/03/26 # NOTES: # if [ $# -lt 2 ] then echo "" echo "usage: netapp_sm_monitor.sh <filer> <srcvol> [-v]" echo "" exit fi i=0; j=0; while true; do j=$i; i=`ssh $1 "snapmirror status -l $2" | grep Progress | awk '{print $2}'| cut -d \( -f2`; if [ $j -eq 0 ]; then sleep 1m; continue; fi; if [ "$3" == "-v" ] then echo -n "$i-$j = "; fi echo "scale=2;($i-$j)/(1024*1024)" | bc | tr '\n' ' '; echo "GB/min"; sleep 1m; done;
monitor NetApp RAID reconstruction
# while true; do echo $(ssh fs03 “sysconfig -r” |grep reconstruction | grep RAID | awk ‘{print $5}’) `date`; sleep 5m; done;
31% Wed Oct 23 15:29:42 PDT 2013
monitor Apache memory usage
When looking at a webserver for memory usage, it’s important to consider the VSZ and RSS memory usage.
This little one liner gets the Total and Average VSZ and RSS usage as well as thread count, and prints those statistics every 5 seconds:
# while true; do ps auxfww | grep apache | grep -v -e cronolog -e grep | awk '{ vsum+=$5; rsum+=$6 } END { print "VSZ:", vsum, "(", vsum/NR, ") RSS:", rsum, "(", rsum/NR, ") Procs:", NR }'; sleep 5; done;
VSZ: 9896272 ( 341251 ) RSS: 1716216 ( 59179.9 ) Procs: 29
VSZ: 9547608 ( 340986 ) RSS: 1650100 ( 58932.1 ) Procs: 28
VSZ: 9546328 ( 340940 ) RSS: 1649044 ( 58894.4 ) Procs: 28
VSZ: 9861976 ( 340068 ) RSS: 1687968 ( 58205.8 ) Procs: 29
VSZ: 9868632 ( 340298 ) RSS: 1694496 ( 58430.9 ) Procs: 29
VSZ: 9853272 ( 339768 ) RSS: 1679112 ( 57900.4 ) Procs: 29
VSZ: 9853272 ( 339768 ) RSS: 1679264 ( 57905.7 ) Procs: 29
^C
#
So there are around 29 threads running right now on this server. The threads are using an average of 340MB per thread VSZ, and 59MB per thread RSS. The total of around 1.7GB of RSS looks good, on a machine with 8G physical memory.
monitor host for slow ping times
When there is intermittent network latency to a host, it’s important to monitor a it for a pattern. Using ping can help narrow down what is causing the latency. VMWare load, bandwidth limitations, employee work patterns, backups, and many other sources could be the cause of the latency.
while true; do j=`ping <slowhost> -i1 -c1 2>&1 | grep icmp_req | awk '{print $7}' | cut -d = -f2 | cut -d . -f1`; if [ $j -gt 30 ]; then date | tr '\n' ' '; echo $j; fi; sleep 1s; done;
This does a ping every second, and if it’s over a threshold (30ms in this case) it is considered unacceptable and logged with date.
monitor host for connectivity
Sometimes, you want to be notified if a host goes up or down. Usually Nagios is perfect for this, but in this case I had an internet circuit, and all I cared about was knowing when the ISP deactivated it. Use ping in a loop, make 1 request every second, if ping doesn’t get a response, then send a text message (Verizon number) and stop the loop.
while true; do ping -nc 1 -W 1 5.6.7.8 | grep -q icmp; if [ "$?" == "1" ]; then echo "circuit is down" | mail <10-digit phone number no spaces>@vtext.com; break; fi; sleep 1s; done;
I also use the converse of this method when I want to know when a new circuit comes up.
use command line to add pool and virtual server to f5 BigIP load balancer
Using the bigpipe
cli command (or it’s alias “b
“) to add pools and virtual servers, can save you hundreds of clicks. This is a very old version of a bigip.
# uname -r
BIG-IP 4.5.14
This creates a pool named myserverpool and adds a single member to it:
# b pool myserverpool {member 172.16.11.201:80}
To add many servers just use a while loop:
# i=202; while [ "$i" -lt "237" ]; do b pool myserverpool add \{ member 172.16.11.$i:80 \}; i=$(($i+1)); done;
Repeat for https:
# b pool myserverpool_ssl {member 172.16.11.201:443}
# i=202; while [ "$i" -lt "237" ]; do b pool myserverpool_ssl add \{ member 172.16.11.$i:443 \}; i=$(($i+1)); done;
Now add health checks:
# i=201; while [ "$i" -lt "237" ]; do b node 172.16.11.$i:80 monitor use http; i=$(($i+1)); done;
# i=201; while [ "$i" -lt "237" ]; do b node 172.16.11.$i:443 monitor use https; i=$(($i+1)); done;
And finally create the virtual servers, pointing traffic to the corresponding pools:
# b virtual 5.6.7.8:80 use pool myserverpool
# b virtual 5.6.7.8:443 use pool myserverpool_ssl