#!/bin/bash
# SARG - Daily/Weekly/Monthly Squid usage reports creation tool
# Written by Ugo Viti <ugo.viti@initzero.it>

# Thanks for enanchements to:
# - martijn
# - Stas Degteff https://sourceforge.net/users/stas_degteff/

VER=debian-sarg-2.3.10-2-with-lastlog

## What is this?
# sarg-reports (this file) is a simple bash script written to automate
# the SARG (a powerful squid log analyzer) reports and log management.
# Sarg it self, provide to end user a generic interface to create
# reports based on squid access log (begin of log to current date).
# sarg-reports (this script) is useful because it allow you to easly
# create and manage Daily, Weekly and Monthly reports.
# Try it, within 5 minutes you will be ready to rule :-)
# using sarg-reports is very easy, read the following 3 steps to know how

## Requirements
# a) An unix system with bash shell (like GNU/Linux, FreeBSD, etc...)
# b) Squid - http://www.squid-cache.org
# c) Sarg - http://web.onda.com.br/orso/sarg.html


##
## Installation guide and configuration parameters
##

# 1) Download Squid and Sarg, Install, Configure and Tune
#    they before continue reading

# 2) In root crontab (crontab -e) insert the following lines:
#    (the today report creation time depend mostly of your squid server
#    load average, tune it):
#
#    --- BEGIN ROOT CRONTAB ---
#    PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
#    00 08-18/1 * * * sarg-reports today
#    00 00      * * * sarg-reports daily
#    00 01      * * 1 sarg-reports weekly 
#    30 02      1 * * sarg-reports monthly
#    --- END ROOT CRONTAB ---
#
#    REMEMBER: if you use logrotate, configure it to retain enough logs to ensure availability
#              of data for the longest period for which you need a report.
#              See https://sourceforge.net/p/sarg/wiki/Keeping%20enough%20log%20to%20process/
#              This version of sarg-reports will use the log file defined in sarg.conf and also
#              prior log files which have been rotated out of that location

# 3) Customize the following variables:
#    (Please, configure accurately the sarg-reports.conf file before)
#
#           (SARG) The sarg executable location
#         (CONFIG) The sarg main configuration file location
#        (HTMLOUT) Location where will be saved the reports
#   (LASTLOGDAILY) Number of previous daily reports to keep (affects "today" and "manual" logs too)
#  (LASTLOGWEEKLY) Number of previous weekly reports to keep
# (LASTLOGMONTHLY) Number of previous monthly reports to keep
#      (PAGETITLE) The title of main index page
#        (LOGOIMG) Image logo to view in main index page
#       (LOGOLINK) HTTP web page link of logo
#          (DAILY) Word 'daily' translation, translate it to your language
#         (WEEKLY) Word 'weekly' translation, translate it to your language
#        (MONTHLY) Word 'monthly' translation, translate it to your language
#    (EXCLUDELOG1) Exclude text from cron emails
#          +       (normally, sarg, during cron activity, if it don't find any valid records,
#    (EXCLUDELOG2) it will output an error message (usually on 'today' reports).
#                  I don't want to be warned by email about this, so, i wrote the 'text'
#                  that will be never logged.
#                  This is useful to receive email of real problems only (enjoy that)
#
# ** Debian specific-change **
#    (EXCLUDELOG3) Use this parameter to filter the covered period message

[ -f /etc/sarg/sarg-reports.conf ] || exit 0

. /etc/sarg/sarg-reports.conf

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
## Italian Language
#       SARG=/usr/bin/sarg
#     CONFIG=/etc/sarg/sarg.conf
#    HTMLOUT=/var/www/html/admin/log/proxy
#  PAGETITLE="Statistiche Proxy di $(hostname)"
#    LOGOIMG=http://www.initzero.it/images/initzero-logo.jpg
#   LOGOLINK=http://www.initzero.it
#      DAILY=Giornaliero
#     WEEKLY=Settimanale
#    MONTHLY=Mensile
#EXCLUDELOG1="SARG: Nessun records trovato."
#EXCLUDELOG2="SARG: Fine"
#EXCLUDELOG3="SARG: Periodo"
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@


#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
## Russian Language
#       SARG="/usr/bin/sarg"
#     CONFIG=/etc/sarg/sarg.conf
#    HTMLOUT=/var/www/sarg-reports
#  PAGETITLE="óÔÁÔÉÓÔÉËÁ ÓÅÒ×ÅÒÁ ÐÒÏËÓÉ Squid"   # russian koi8-r
#    LOGOIMG=http://litek.ru/images/logotop.gif
#   LOGOLINK=http://litek.ru
#      DAILY="åÖÅÄÎÅ×ÎÁÑ"     # russian koi8-r
#     WEEKLY="åÖÅÎÅÄÅÌØÎÁÑ"   # russian koi8-r
#    MONTHLY="åÖÅÍÅÓÑÞÎÁÑ"    # russian koi8-r
#EXCLUDELOG1="SARG: Records in file:"
#EXCLUDELOG2="SARG: ïÔÞÅÔ ÕÓÐÅÛÎÏ ÓÇÅÎÅÒÉÒÏ×ÁÎ ×" # sarg.conf: language Russian_koi8
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@


######################################################################
## The configuration is end, so don't touch anything bellow

# TEMP Files
TMPFILE=`/bin/mktemp`
 ERRORS="${TMPFILE}.errors"
TMPDIR=`/bin/mktemp -d`

# Date Calc
MANUALDATE=$2
case "$(uname)" in
"FreeBSD")
         TODAY=$(date +%d/%m/%Y)
     YESTERDAY=$(date -v-1d +%d/%m/%Y)
       WEEKAGO=$(date -v-1w +%d/%m/%Y)
      MONTHAGO=$(date -v-1m +01/%m/%Y)-$(date -v-1m +31/%m/%Y)
   ;;
"OpenBSD")
         TODAY=$(date +%d/%m/%Y)
     YESTERDAY=$(date -r $((`date +%s` - 86400 )) +%d/%m/%Y)
       WEEKAGO=$(date -r $((`date +%s` - 604800)) +%d/%m/%Y)
      MONTHAGO=$(perl -e '@t=localtime(time); $y=$t[4]==0?$t[5]+1899:$t[5]+1900; $m=$t[4]==0?12:$t[4]; print "1/$m/$y-",$m==2?$y%4>0?28:29:$m==4||$m==6||$m==9||$m==11?30:31 ,"/$m/$y\n";')
   ;;
*)
         TODAY=$(date --date "today" +%d/%m/%Y)
     YESTERDAY=$(date --date "1 day ago" +%d/%m/%Y)
       WEEKAGO=$(date --date "1 week ago" +%d/%m/%Y)
      MONTHAGO=$(date --date "1 month ago" +01/%m/%Y)-$(date --date "1 month ago" +31/%m/%Y)
   ;;
esac

# Fix for Red Hat 9 systems and coreutils prior to 5.0 version
export LC_ALL=C

# Main index.html creation
create_index_html ()
{
  echo -e "\
  <html>\n\
  <head>\n\
  <title>$PAGETITLE</title>\n\
  </head>\n\
  <body>\n\
  <div align=center>\n\
    <a href=$LOGOLINK><img border=0 src=$LOGOIMG></a>\n\
    <table border=0 cellspacing=6 cellpadding=7>\n\
      <tr>\n\
        <th align=center nowrap><b><font face=Arial size=4 color=green>$PAGETITLE</font></b></th>\n\
      </tr>\n\
      <tr>\n\
        <td align=center bgcolor=beige><font face=Arial size=3><a href=$DAILY>$DAILY</a></font></td>\n\
      </tr>\n\
      <tr>\n\
        <td align=center bgcolor=beige><font face=Arial size=3><a href=$WEEKLY>$WEEKLY</a></font></td>\n\
      </tr>\n\
      <tr>\n\
        <td align=center bgcolor=beige><font face=Arial size=3><a href=$MONTHLY>$MONTHLY</a></font></td>\n\
      </tr>\n\
    </table>\n\
  </div>\n\
  </body>\n\
  </html>" > $HTMLOUT/index.html
}

reports_autorm () {
  BAKLIST="${TMPFILE}.BAKLIST"
  BAKLISTTOLEAVELIST="${BAKLIST}.TOLEAVE.LIST"
  BAKLISTTOLEAVEFIND="${BAKLIST}.TOLEAVE.FIND"
  touch $BAKLIST $BAKLISTTOLEAVELIST $BAKLISTTOLEAVEFIND

  cd $DESTINATION

  find -name "*$BAKNAME*" -maxdepth 1 | sed -e 's/.\///' | grep -wv "." | sort 1> $BAKLIST

  let TOLEAVEDAYS="$DAYBAK+1"

  if [ $TOLEAVEDAYS -ge "0" ]
    then
      cat $BAKLIST | tail -n $TOLEAVEDAYS > $BAKLISTTOLEAVELIST
      for BAK in $(cat $BAKLISTTOLEAVELIST)
        do
          echo "! -name $BAK" >> $BAKLISTTOLEAVEFIND
        done
  fi

  CURRENTH=$(date +%H)
  CURRENTM=$(date +%M)
  if [ ${CURRENTH:0:1} = 0 ]
  then
    CURRENTH=${CURRENTH:1:1}
  fi
  if [ ${CURRENTM:0:1} = 0 ]
    then
      CURRENTM=${CURRENTM:1:1}
  fi
  let MINUTES="((($DAYBAK*24)+$CURRENTH)*60)+$CURRENTM+1"

  # Purging old backups, only if backup return "OK" status
  find $DESTINATION -name "*$BAKNAME*" $(cat $BAKLISTTOLEAVEFIND) -maxdepth 1 -mmin +$MINUTES -exec rm -rf {} \;
  #find $DESTINATION $(cat $BAKLISTTOLEAVEFIND) -maxdepth 1 -mmin +$MINUTES -exec rm -rf {} \;
}

# Functions
exclude_from_log ()
{
  cat $ERRORS | grep -v "$EXCLUDELOG1" | grep -v "$EXCLUDELOG2" | grep -v "$EXCLUDELOG3"
  rm -f $TMPFILE*
  rm -rf $TMPDIR
}

manual ()
{
  if [ -n "$LASTLOGDAILY" ]
    then
      LASTLOG="--lastlog=$LASTLOGDAILY"
  fi
  DAILYOUT=$HTMLOUT/$DAILY
  mkdir -p $DAILYOUT
  create_index_html
  if [ -z "$MANUALDATE" ]
    then
      echo "No date given, please specify a valid date (DD/MM/YYYY)"
    else
      $SARG -f $CONFIG -w $TMPDIR -d $MANUALDATE -o $DAILYOUT $LASTLOG ${@} ${ACCESS_LOG}*
  fi
}

today ()
{
  if [ -n "$LASTLOGDAILY" ]
    then
      LASTLOG="--lastlog=$LASTLOGDAILY"
  fi
  DAILYOUT=$HTMLOUT/$DAILY
  mkdir -p $DAILYOUT
  create_index_html
  $SARG -f $CONFIG -w $TMPDIR -d $TODAY -o $DAILYOUT $LASTLOG ${@} ${ACCESS_LOG}* >$ERRORS 2>&1
  exclude_from_log
}

daily ()
{
  if [ -n "$LASTLOGDAILY" ]
    then
      LASTLOG="--lastlog=$LASTLOGDAILY"
  fi
  DAILYOUT=$HTMLOUT/$DAILY
  mkdir -p $DAILYOUT
  create_index_html
  $SARG -f $CONFIG -w $TMPDIR -d day-1 -o $DAILYOUT $LASTLOG ${@} ${ACCESS_LOG}* >$ERRORS 2>&1
  exclude_from_log
}

weekly ()
{
  if [ -n "$LASTLOGWEEKLY" ]
    then
      LASTLOG="--lastlog=$LASTLOGWEEKLY"
  fi
  WEEKLYOUT=$HTMLOUT/$WEEKLY
  mkdir -p $WEEKLYOUT
  create_index_html
  $SARG -f $CONFIG -w $TMPDIR -d week-1 -o $WEEKLYOUT $LASTLOG ${@} ${ACCESS_LOG}* >$ERRORS 2>&1
  exclude_from_log
}

monthly ()
{
  if [ -n "$LASTLOGMONTHLY" ]
    then
      LASTLOG="--lastlog=$LASTLOGMONTHLY"
  fi
  MONTHLYOUT=$HTMLOUT/$MONTHLY
  mkdir -p $MONTHLYOUT
  create_index_html
  $SARG -f $CONFIG -w $TMPDIR -d month-1 -o $MONTHLYOUT $LASTLOG ${@} ${ACCESS_LOG}* >$ERRORS 2>&1
  exclude_from_log
}

ACCESS_LOG=$(sed -ne 's|^access_log[[:space:]]*||p' /etc/sarg/sarg.conf)

case $1 in
    manual)
	shift 1
        manual "${@}"
        ;;
    today)
	shift 1
        today "${@}"
        ;;
    daily)
	shift 1
        daily "${@}"
        ;;
    weekly)
	shift 1
        weekly "${@}"
        ;;
    monthly)
	shift 1
        monthly "${@}"
        ;;
     *)
        echo "SARG - Daily / Weekly / Monthly - Squid proxy usage reports creation tool"
        echo "Written by Ugo Viti <ugo.viti@initzero.it>"
        echo "Version: $VER"
        echo
        echo "Usage: $0 [OPTIONS]"
        echo
        echo "Allowed options:"
        echo "    manual,  Create Manual report"
        echo "     today,  Create Today report"
        echo "     daily,  Create Daily report"
        echo "    weekly,  Create Weekly report"
        echo "   monthly,  Create Monthly report"
        exit 0
esac


## HISTORY:
# debian-sarg-2.3.10-2-with-lastlog - add lastlog option to keep different numbers of previous reports per type, private temporary directory
# debian-sarg-2.3.10-2 - use all rotated log files, pass through options from command line
# debian-sarg-? - added reports_autorm function (not used)
# ... - ?
# 20050502 - Stas Degteff added support for non latin Charset and added support for OpenBSD
# 20030826 - FreeBSD support (thanks to martijn to let me coding on your FreeBSD server :-))
# 20030715 - Some cleanups
# 20030623 - Manual report creation
# 20030620 - Main Index creation
# 20030619 - Solved 'sort' bug on Red Hat 9 systems
# 20030618 - First Version


## TODO:
# - Smarty weekly recognition...
#   Like "begin of last week to end of last week",
#   doesn't like this script do: "7 days ago to yesterday"
# - Monthly recognition isn't so elegant (is very ugly, i know)
# - Suggestions are welcome :-)
