#!/bin/bash ############################################################################## ## IMPORTANT: This script is not supported by Scalix. Use at your own risk. ## ############################################################################## # # NAME: ommaint - Perform periodic maintenance on Scalix Server # # SYNOPSIS: ommaint -frequent | -daily | -weekly # # DESCRIPTION: # Perform Scalix regular maintenance. I thought of everything I make # Scalix do on a periodic basis and threw it in here. It takes frequent, # daily, and weekly maintenance and puts them in a single script so that # it's easier to e-mail around. You typically cron this to run as follows: # # # minute hour monthday month weekday command # 00,30 * * * * /usr/local/bin/ommaint -frequent # 01 0 * * * /usr/local/bin/ommaint -daily # 15 2 * * 0 /usr/local/bin/ommaint -weekly # # Exactly when you run everything doesn't matter. Frequent is for frequent # ops such as checking for aborted queues or other errors. Daily empties # the users' wastebaskets, rolls the audit logs, and does backups. Weekly # runs omscan. You might be able to throw omscan in the daily part if you # have a small installation. # # Over time you'll want to adapt this to your preferences. This is intended # to provide a starting point for system admins new to Scalix. In # particular, be sure to take a look at the backup section, since you'll # probably want to modify that to your preferred way of doing backups. For # simplicity I just use tar here because it's simple and it always exists. # # PLATFORMS TESTED: RedHat Enterprise Linux 3.0 # # SUPPORT: None. However, if you make a useful addition that you think others # can benefit from, or if you port it to a different platform, or # if you find *and fix* a bug, please send your modifications # to Scalix Support and we will update the original. # # # SECTION: General Configuration Section # # General configuration for this script # # Address to mail reports to MAIL_REPORTS="root" # # MAIL_REPORTS="custlogs@scalix.com" # MAIL_REPORTS="custlogs@scalix.com,root" # # You may enable this script to automatically send log file to Scalix Corp. Uncomment the above line such that the log files # generated by this script will be sent to "custlogs@scalix.com". Please note that multiple email address may be specified, # is seperated by a comma. # # By uncommenting the code line above, you enable the automatic transmission of your system log files to Scalix. # When a problem arises with your system and you notify Scalix support of your problem, Scalix's possession of your # log files may assist Scalix in making a prompt and accurate diagnosis of your problem. However, enabling automatic # transmission of your log files does not impose any obligation on Scalix. Scalix does not regularly review the log # files it receives. Scalix does not monitor its customers' system performance. And Scalix does not, without a prior # support request from a customer, proactively evaluate system performance and notify customers of suspected problems. # If you enable automatic transmission of your log files to Scalix, SCALIX ASSUMES NO OBLIGATIONS ADDITIONAL TO THOSE # OBLIGATIONS WHICH ALREADY EXIST UNDER THE WRITTEN CONTRACTUAL RELATIONSHIP BETWEEN YOU AND SCALIX. # Device to use for backups BACKUP_DEVICE=/dev/rmt0 # Scalix bin directory SX_BIN_DIR=/opt/scalix/bin # ---------------------------------------------------------------------- # SECTION 1: FREQUENT OPERATIONS. # # Usage: ommaint -frequent # # These operations should be done at frequent intervals, no less than # once/hour. Typically they do things such as checking for problems # such as running out of disk space - things that should be done # frequently. They also include operations that do not need to be # done frequently, but also don't require much time to complete. # ommaint_frequent () { MAINTLOG=/tmp/ommaint.$$ STDERR=/tmp/ommaint.$$.stderr STDOUT=/tmp/ommaint.$$.stdout rm -f $MAINTLOG $STDERR $STDOUT touch $MAINTLOG $STDERR $STDOUT # ACTION: Check Disk Space Utilization # Notify administrator if disk space utilization on any filesystem >= 80% # df -t ext3 | awk '{if (int($5)>=80) {print $0}}' > $STDOUT if [ -s $STDOUT ] then echo "These filesystems have >= 80% disk space utilization:" >> $MAINTLOG echo "--------------------------------------------------" >> $MAINTLOG df -t ext3 | head -1 >> $MAINTLOG cat $STDOUT >> $MAINTLOG fi # ACTION: Check Inode Utilization # Notify administrator if inode utilization on any filesystem >= 80% # df -i -t ext3 | awk '{if (int($5)>=80) {print $0}}' > $STDOUT if [ -s $STDOUT ] then echo "These filesystems have >= 80% inode utilization:" >> $MAINTLOG echo "--------------------------------------------------" >> $MAINTLOG df -i -t ext3 | head -1 >> $MAINTLOG cat $STDOUT >> $MAINTLOG fi # ACTION: Check the Error Queues # Include the results in the report to the system admin only if there # are some messages in the ERROR queues # MULTI_SERVER_TRUE=`$SX_BIN_DIR/sxlicense |grep -c '^System Class: Multi-Server'` if [ $MULTI_SERVER_TRUE -gt 0 ] then S_ERROR='SMERR' fi for queue in ERROR $S_ERROR do $SX_BIN_DIR/omstat -q $queue >$STDOUT 2>$STDERR if [ "$(cat $STDERR)" != "omstat : There are no messages on the queue" ] then echo "Messages on the Scalix $queue Queue" >> $MAINTLOG echo "----------------------------------------------------" >> $MAINTLOG cat $STDERR >> $MAINTLOG cat $STDOUT >> $MAINTLOG fi done # ACTION: Check the Scalix Queues # Report to the administrator if the number of messages queued up for # delivery on any of the standard queues exceeds 10. # services="service router|local delivery|internet mail gateway|sendmail interface" $SX_BIN_DIR/omstat -s | grep -Ei "$services" >$STDOUT 2>$STDERR cat $STDOUT | while read line do queue=`echo $line | awk '{NF=NF-3;print $0}'` msgcount=`echo $line | awk '{print $NF}'` if [ $msgcount -gt 10 ] then echo "Messages on the Scalix $queue Queue = $msgcount" fi done # ACTION: Check for aborted services # Report to the administrator any Scalix services # which have become aborted. Attempt to restart the service # $SX_BIN_DIR/omstat -s -j | grep -i aborted > $STDOUT if [ -s $STDOUT ] then echo "The following services are ABORTED:" >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG cat $STDOUT >> $MAINTLOG echo "" >> $MAINTLOG # ACTION: Copy content of Fatal Log to output file if [ -s /var/opt/scalix/logs/fatal ] then echo "-------------------------------------------" >> $MAINTLOG echo "The following text is the output from the fatal log" >> $MAINTLOG echo "" >> $MAINTLOG cat /var/opt/scalix/logs/fatal >> $MAINTLOG echo "End of File - Fatal Long" >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "" >> $MAINTLOG # ACTION: Parse the output line and find the service name that is aborted # do an omshowlog for that service and capture the output # do an omon on the service and try and restart it cat $STDOUT | tr " " "_" | while read line do queue=`echo $line | awk '{print $3}' | tr "_" " "` echo "The following text is the output from omshowlog -s ""$queue"" -l 99" >> $MAINTLOG $SX_BIN_DIR/omshowlog -s "$queue" -l 99 >> $MAINTLOG echo "End of output from omshowlog" >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "Attempting to start the aborted service with omon -s ""$queue""" >> $MAINTLOG $SX_BIN_DIR/omon -s "$queue" >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG done fi fi # ACTION: Check for aborted daemons # Report to the administrator any Scalix daemons # which have become aborted. Attempt to restart the service # $SX_BIN_DIR/omstat -a -j | grep -i aborted > $STDOUT if [ -s $STDOUT ] then echo "The following daemons are ABORTED:" >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG cat $STDOUT >> $MAINTLOG echo "" >> $MAINTLOG # ACTION: Copy content of Fatal Log to output file if [ -s /var/opt/scalix/logs/fatal ] then echo "-------------------------------------------" >> $MAINTLOG echo "The following text is the output from the fatal log" >> $MAINTLOG echo "" >> $MAINTLOG cat /var/opt/scalix/logs/fatal >> $MAINTLOG echo "End of File - Fatal Long" >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "" >> $MAINTLOG # ACTION: Parse the output line and find the daemon name that is aborted # do an omon on the daemon and try and restart it cat $STDOUT | tr " " "_" | while read line do daemon=`echo $line | awk '{print $3}' | tr "_" " "` echo "The following text is the output from omshowlog -s ""$daemon"" -l 99" >> $MAINTLOG $SX_BIN_DIR/omshowlog -s "$daemon" -l 99 >> $MAINTLOG echo "End of output from omshowlog" >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "Attempting to start the aborted daemon with omon -a ""$daemon""" >> $MAINTLOG $SX_BIN_DIR/omon -a "$daemon" >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "------------------------------------------" >> $MAINTLOG done fi fi # ACTION: Check that the CheckPort.pl script exists. If not, create it # PERLSCRIPT=./PortCheck.pl if [ ! -s $PERLSCRIPT ] then echo '#!/usr/bin/perl' >> $PERLSCRIPT echo 'use IO::Socket::INET;' >> $PERLSCRIPT echo '' >> $PERLSCRIPT echo 'eval {' >> $PERLSCRIPT echo 'local $SIG{ALRM} = sub { die "Timed Out"; };' >> $PERLSCRIPT echo 'alarm 10;' >> $PERLSCRIPT echo '' >> $PERLSCRIPT echo '$socket = IO::Socket::INET->new(PeerAddr => $ARGV[0],' >> $PERLSCRIPT echo ' PeerPort => $ARGV[1],' >> $PERLSCRIPT echo ' Proto => "tcp",' >> $PERLSCRIPT echo ' Type => SOCK_STREAM)' >> $PERLSCRIPT echo 'or die "Couldnt open port $ARGV[0]:$ARGV[1] : $!\n";' >> $PERLSCRIPT echo '' >> $PERLSCRIPT echo '$answer = <$socket>;' >> $PERLSCRIPT echo 'print $answer;' >> $PERLSCRIPT echo 'close($socket);' >> $PERLSCRIPT echo 'alarm 0;' >> $PERLSCRIPT echo '};' >> $PERLSCRIPT echo 'alarm 0;' >> $PERLSCRIPT echo 'if ($@ =~ /Timed Out/) {' >> $PERLSCRIPT echo ' print "Error: Timeout on port $ARGV[0]:$ARGV[1]\n";' >> $PERLSCRIPT echo ' exit 1;' >> $PERLSCRIPT echo '};' >> $PERLSCRIPT echo 'if ($@ =~ /Couldnt open port/) {' >> $PERLSCRIPT echo ' print "$@";' >> $PERLSCRIPT echo ' exit 0;' >> $PERLSCRIPT echo '};' >> $PERLSCRIPT echo 'if ($@ ne "") {' >> $PERLSCRIPT echo ' exit -1;' >> $PERLSCRIPT echo '}' >> $PERLSCRIPT echo 'exit -2;' >> $PERLSCRIPT chmod +x $PERLSCRIPT fi # ACTION: Check the status of port 25, 110 and 143. # Report to the administrator any Scalix ports that are not responding # Attempt to restart the offending port # echo "--Performing Port Check on localhost:25-----" >> $STDOUT $PERLSCRIPT `hostname` 25 >> $STDOUT if [ $? -eq 1 ] then cat $STDOUT >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG # ACTION: Copy content of Fatal Log to output file if [ -s /var/opt/scalix/logs/fatal ] then echo "-------------------------------------------" >> $MAINTLOG echo "The following text is the output from the fatal log" >> $MAINTLOG echo "" >> $MAINTLOG cat /var/opt/scalix/logs/fatal >> $MAINTLOG echo "End of File - Fatal Long" >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "" >> $MAINTLOG fi daemon="SMTP Relay" echo "The following text is the output from omshowlog -s ""$daemon"" -l 99" >> $MAINTLOG $SX_BIN_DIR/omshowlog -s "$daemon" -l 99 >> $MAINTLOG 2> $STDERR cat $STDERR >> $MAINTLOG echo "End of output from omshowlog" >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "Attempting to stop the SMTP service with omoff -d0 -s ""$daemon""" >> $MAINTLOG $SX_BIN_DIR/omoff -d0 -a "$daemon" >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "Attempting to start the SMTP service with omon -s ""$daemon""" >> $MAINTLOG $SX_BIN_DIR/omon -a "$daemon" >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG fi echo "--Performing Port Check on localhost:110---" >> $STDOUT $PERLSCRIPT `hostname` 110 >> $STDOUT if [ $? -eq 1 ] then cat $STDOUT >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG # ACTION: Copy content of Fatal Log to output file if [ -s /var/opt/scalix/logs/fatal ] then echo "-------------------------------------------" >> $MAINTLOG echo "The following text is the output from the fatal log" >> $MAINTLOG echo "" >> $MAINTLOG cat /var/opt/scalix/logs/fatal >> $MAINTLOG echo "End of File - Fatal Long" >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "" >> $MAINTLOG fi queue="POP3 interface" echo "The following text is the output from omshowlog -s ""$queue"" -l 99" >> $MAINTLOG $SX_BIN_DIR/omshowlog -s "$queue" -l 99 >> $MAINTLOG 2> $STDERR cat $STDERR >> $MAINTLOG echo "End of output from omshowlog" >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "Attempting to stop the POP3 service with omoff -d0 -s ""$queue""" >> $MAINTLOG $SX_BIN_DIR/omoff -d0 -s "$queue" >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "Attempting to start the POP3 service with omon -s ""$queue""" >> $MAINTLOG $SX_BIN_DIR/omon -s "$queue" >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG fi echo "--Performing Port Check on localhost:143-----" >> $STDOUT $PERLSCRIPT `hostname` 143 >> $STDOUT if [ $? -eq 1 ] then cat $STDOUT >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG # ACTION: Copy content of Fatal Log to output file if [ -s /var/opt/scalix/logs/fatal ] then echo "-------------------------------------------" >> $MAINTLOG echo "The following text is the output from the fatal log" >> $MAINTLOG echo "" >> $MAINTLOG cat /var/opt/scalix/logs/fatal >> $MAINTLOG echo "End of File - Fatal Long" >> $MAINTLOG echo "-------------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "" >> $MAINTLOG fi daemon="IMAP Server Daemon" echo "The following text is the output from omshowlog -s ""$daemon"" -l 99" >> $MAINTLOG $SX_BIN_DIR/omshowlog -s "$daemon" -l 99 >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "End of output from omshowlog" >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "" >> $MAINTLOG echo "Attempting to stop the IMAP service with omoff -d0 -s ""$daemon""" >> $MAINTLOG $SX_BIN_DIR/omoff -d0 -a "$daemon" >> $MAINTLOG 2> $STDERR echo "" >> $MAINTLOG cat $STDERR >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG echo "Attempting to start the IMAP service with omon -s ""$daemon""" >> $MAINTLOG $SX_BIN_DIR/omon -a "$daemon" >> $MAINTLOG 2> $STDERR cat $STDERR >> $MAINTLOG echo "-----------------------------------------" >> $MAINTLOG fi # ACTION: Add missing entries to the FREEBUSY # #omshowu -m all | omaddent -d freebusy -x -f - 2>/dev/null # ACTION: Re-build the Sendmail Alias file from the Scalix directory # #cp -f /etc/mail/aliases.head /etc/mail/aliases #/opt/scalix/bin/omsearch -e "internet-addr=*" -s -m s/g/ou1/ou2/internet-addr 2>/dev/null | sed -e "s/S=\(.*\)\/G=\(.*\)\/OU1=\(.*\)\/OU2=\(.*\)\/INTERNET-ADDR=\(.*\)/\2.\1\/\3_\4 : \5/" >> /etc/mail/aliases #newaliases 2>&1 >/dev/null if [ -s $MAINTLOG ] then mail -s "`hostname` - Scalix Frequent Maintenance Report" $MAIL_REPORTS < $MAINTLOG fi rm -f $MAINTLOG $STDERR $STDOUT } # ---------------------------------------------------------------------- # SECTION 2: DAILY OPERATIONS. # # Usage: ommaint -daily # # These operations should be done at daily intervals. Typically they do # things such as maintaining log files so that log files do not grow # to consume all available disk space. # # Daily operations should be performed around midnight. This is for two # reasons. First, log files should be rolled around midnight so that # audit.Wed is indeed Wednesday's data. Second, some daily maintenance # operations are CPU and disk intensive, and should not coincide with # periods of heavy system load. System load is usually highest during # working hours when users actively using the server. # ommaint_daily () { DAY=`date '+%a'` MAINTLOG=/var/opt/scalix/logs/ommaint.$DAY rm -f $MAINTLOG touch $MAINTLOG # ACTION: Remove items deleted > 30 days ago from users' trash folders # # CUSTOMIZATIONS AVAILABLE: # omtidyallu can also used to remove outdated items at customizable # periods from the trash, inbox, outbox, and/or folders. It's also possible # to create different retention schedules for individual users using # omtidyu, and many other customizations are available. See # "man omtidyallu" for these and other configurations. # #/opt/scalix/bin/omtidyallu -w 30 -d 2>&1 >> $MAINTLOG # ACTION: Age the Scalix Audit log # AUDFIL=/var/opt/scalix/logs/audit DAY=`date '+%a'` SAVFIL="$AUDFIL.$DAY" rm -f $SAVFIL 2>&1 >> $MAINTLOG mv -f $AUDFIL $SAVFIL 2>&1 >> $MAINTLOG touch $AUDFIL 2>&1 >> $MAINTLOG chown scalix $AUDFIL 2>&1 >> $MAINTLOG chgrp scalix $AUDFIL 2>&1 >> $MAINTLOG chmod 660 $AUDFIL 2>&1 >> $MAINTLOG # ACTION: Age the Sendmail log file # #MAILLOG=/var/log/maillog #DAY=`date '+%a'` #SAVFIL="$MAILLOG.$DAY" #rm -f $SAVFIL 2>&1 >> $MAINTLOG #mv -f $MAILLOG $SAVFIL 2>&1 >> $MAINTLOG #touch $MAILLOG 2>&1 >> $MAINTLOG #chown root $MAILLOG 2>&1 >> $MAINTLOG #chgrp root $MAILLOG 2>&1 >> $MAINTLOG #chmod 600 $MAILLOG 2>&1 >> $MAINTLOG # ACTION: Backup Scalix # # WARNING: # In order to guarantee Scalix database integrity, Scalix backups # must be made from an offline copy of the data. This is because a # change to the database is generally made by modifying multiple files # simultaneously. If, during a backup, one of these files is modified # after it is backed up, but the others are modified before they are # backed up, than the backup will be corrupt. The options for backing # up are therefore: # 1. Shut down Scalix, back up, Restart # Advant: Backup not corrupt. Simple. Always works # DisAdv: System will be unavailable during backup. # 2. Use advanced filesystem such as ReiserFS, XFS, or Veritas # to make a "snapshot" filesystem. A snapshot filesystem holds # the changes made since the backup started, and allows the # backup program to access a "snapshot" of the system at the # time the backup started. # Advant: Backup not corrupt. No downtime. # Disadv: Requires advanced FS. Requires approximately extra # 10% diskspace to hold snapshot information. # 3. Backup Scalix online from online data set. The restore # procedure will need to include commands to uncorrupt the # database. Those commands take time to run. This generally # works, but it is not guaranteed to work and is "unsupported". # The command to uncorrupt a database is omscan(8). # Advant: No downtime. Simple. # Disadv: Lengthy, unreliable restore. Unsupported. # # This backup script does option #1, shut down to backup, so that # it will work even if you do not have an advanced filesystem. If # you do have an advanced filesystem, comment out the default backup # script and modify the steps below to build a backup script based # on a snapshot filesystem. # # This sample backup script does a full backup of Scalix data in # /var/opt/scalix, and it uses tar. Scalix does not care if you # do full or partials, if you backup /, or which backup program you use. # However, please take note of the WARNING above. # # Standard backup script: # #/opt/scalix/bin/omshut 2>&1 >> $MAINTLOG #tar -cf $BACKUP_DEVICE /var/opt/scalix 2>&1 >> $MAINTLOG #/opt/scalix/bin/omrc 2>&1 >> $MAINTLOG # # Backup script template for advanced filesystems: # # # /opt/scalix/bin/omsuspend -s 300 & # Pause OM while making snapshot # sync # This is a must! # create_snapshot /var/opt/scalix /varoptomsnap # Create snapshot FS # /opt/scalix/bin/omsuspend -r # Unpause OM after snapshot ready # tar -cf $BACKUP_DEVICE /varoptomsnap # cancel_snapshot /varoptomsnap if [ -s $MAINTLOG ] then mail -s "`hostname` - Scalix Daily Maintenance Report" $MAIL_REPORTS < $MAINTLOG rm -f $MAINTLOG fi } # ---------------------------------------------------------------------- # SECTION 3: WEEKLY OPERATIONS. # # Usage: ommaint -weekly # # These operations should be done at weekly intervals. Typically they do # things that are not necessary to perform daily and take up so much # processing power that they should only be run infrequently and when # when the system is lightly utilized (such as during a weekend). # ommaint_weekly () { # ACTION: Scan the Scalix Message Store for corruptions, and e-mail a # report to the system administrator # # CONFIGURATION OPTIONS: # See the manual entry for omscan(8) for more options. Unless the system # is very large, you can scan it daily if you prefer. However, daily # scans take up CPU and I/O bandwidth and are generally not necessary. # $SX_BIN_DIR/omscan -a 2>&1 > /var/opt/scalix/logs/omscan.out sed -e 's/^/ /' /var/opt/scalix/logs/omscan.out | mail -s "`hostname` - Scalix Weekly Maintenance Report" $MAIL_REPORTS } usage () { echo "USAGE: ommaint [ -frequent | -daily | -weekly ]" >&2 exit 1 } if [ $# -ne 1 ]; then usage; fi case $1 in -frequent) ommaint_frequent ;; -daily) ommaint_daily ;; -weekly) ommaint_weekly ;; *) usage ;; esac