#!/bin/bash
#########################################################################
#                                                                       #
# This script processed linaro web logs, producing different reports    #
# Reports that can be run:                                              #
# 1) Awffull  (webalizer fork)                                          #
# 2) Webalizer                                                          #
# 3) Webdruid (webalizer fork)                                          #
# 4) Visitors                                                           #
#                                                                       #
# You can run those tools on the full log or a processed log            #
# The full log shows path traveled and other info, the processed log    #
# really only shows what files were downloaded, for releases and        #
# snapshots.linaro.org this is important information.                   #
#                                                                       #
# 1) full log analyses of un unaltered web log                          #
# 2) filtered analyses of the web log of only the .gz and .bz2 files    #
#    and everything else stripped out                                   #
#                                                                       #
# To speed things up we use dnshistory to do reverse DNS                #
# as a time cheat we only run the lookup on www.linaro.org log so       #
# you should always process that log first so your DNS data is fresh    #
#                                                                       #
#########################################################################

# first save where we are
STARTING_LOCATION=`pwd`

#WEB_NAME="snapshots.linaro.org"
#WEB_NAME="releases.linaro.org"
WEB_NAME="www.linaro.org"

TRUE=1
FALSE=0
DEBUG=$TRUE

# house keeping
if [ -z "$WEB_NAME" ] || [ -z "$TRUE" ] ; then
    echo "WEB SITE NAME or other variables NOT SET"
    echo "This script is not designed to be called directly" 
    exit 1
fi

# Which tools are we running, if not set at all set to false
if [ -z "$AWFFULL" ] ; then
    AWFFULL=$FALSE
fi
if [ -z "$WEBALIZER" ] ; then
    WEBALIZER=$FALSE
fi
if [ -z "$WEBDRUID" ] ; then
   WEBDRUID=$FALSE
fi
if [ -z "$VISITORS" ] ; then
    VISITORS=$FALSE
fi

# this allows an external script to set DEBUG, or if it's not set, 
# then set it to false here so the script is run quietly
if [ -z "$DEBUG" ] ; then
    DEBUG=$FALSE
fi 

AWFFULL_FULL_PATH_NAME="awffull.full"
AWFFULL_CONF_FULL_NAME="awffull.conf.full"

AWFFULL_FILTERED_PATH_NAME="awffull.filtered"
AWFFULL_CONF_FILTERED_NAME="awffull.conf.filtered"

AWFFULL_TOOLCHAIN_PATH_NAME="awffull.toolchain"
AWFFULL_CONF_TOOLCHAIN_NAME="awffull.conf.toolchain"

AWFFULL_EXTRA_CMD_LINE_OPTIONS=""

WEBALIZER_FULL_PATH_NAME="webalizer.full"
WEBALIZER_CONF_FULL_NAME="webalizer.conf.full"

WEBALIZER_FILTERED_PATH_NAME="webalizer.filtered"
WEBALIZER_CONF_FILTERED_NAME="webalizer.conf.filtered"

WEBALIZER_TOOLCHAIN_PATH_NAME="webalizer.toolchain"
WEBALIZER_CONF_TOOLCHAIN_NAME="webalizer.conf.toolchain"

WEBALIZER_EXTRA_CMD_LINE_OPTIONS=""

WEBDRUID_FULL_PATH_NAME="webdruid.full"
WEBDRUID_CONF_FULL_NAME="webdruid.conf.full"

WEBDRUID_FILTERED_PATH_NAME="webdruid.filtered"
WEBDRUID_CONF_FILTERED_NAME="webdruid.conf.filtered"

WEBDRUID_TOOLCHAIN_PATH_NAME="webdruid.toolchain"
WEBDRUID_CONF_TOOLCHAIN_NAME="webdruid.conf.toolchain"

WEBDRUID_EXTRA_CMD_LINE_OPTIONS=""



prep_paths_and_variables()
{
# do we want to extract file info and run the log analyzers on only that data
# quite handy for snapshots and releases .linaro.org so we get a better 
# picture of what is downloaded. 
# 1 = true, 0 = false
#if [ $WEB_NAME = "snapshots.linaro.org" ] || [ $WEB_NAME = "releases.linaro.org" ] ; then
#    EXTRACT_GZ_BZ2_FILES=$TRUE
#    EXTRACT_TOOLCHAIN_LOG=$TRUE
#else
    EXTRACT_GZ_BZ2_FILES=$FALSE
    EXTRACT_TOOLCHAIN_LOG=$FALSE
#fi

# this allows an external script to set DO_GEOIP_LOOKUP or DO_REV_DNS_LOOKUP
# as desired but by default they are set to true so the log files have 
# as much data in them as possible.
if [ -z "$DO_GEOIP_LOOKUP" ]  ; then
    DO_GEOIP_LOOKUP=$TRUE
fi
if [ -z "$DO_REV_DNS_LOOKUP" ] ; then
    DO_REV_DNS_LOOKUP=$TRUE
fi

# Load config
source config

#*access*2014*
#if [ $WEB_NAME = "snapshots.linaro.org" ] || [ $WEB_NAME = "releases.linaro.org" ] ; then
#    RAW_LOG_NAME="$WEB_NAME-$PROCESSED_LOG_NAME"
#elif [ $WEB_NAME = "www.linaro.org" ] ; then
#    RAW_LOG_NAME="$PROCESSED_LOG_NAME"
#fi

#########################################################################
#                                                                       #
# only have old logs on www.linaro.org as it's hard to get stuff done   #
# on that machine                                                       #
#                                                                       #
#########################################################################
if [ $WEB_NAME = "www.linaro.org" ] ; then
    OLD_LOG_PATH="$INPUT_PATH/2013"
    OLD_LOG_NAME="$PROCESSED_LOG_NAME"
else
    OLD_LOG_PATH="$INPUT_PATH"
    OLD_LOG_NAME=""
fi

}

write.conf.file ()
{
# $1 = File to write conf into into
# $2 = Output directory for analsys to be written into
# $3 = Info to be written into the web analsys to id what you are looking at

    echo "# DO NOT EDIT THIS FILE, modify analyse-snapshot.sh as it rewrites this file everytime it's run"  > $1
    echo "OutputDir $2"   >> $1
    echo "HTMLPost $3"                                >> $1
    echo "ReportTitle \"Usage Statistics for the $3 of \"" >> $1
    echo "HostName $WEB_NAME"                            >> $1
    echo "ReallyQuiet yes"                               >> $1
    echo "TopSites 100"                                  >> $1
    echo "AllSites yes"                                  >> $1
    echo "TopURLs 100"                                   >> $1
    echo "AllURLs yes"                                   >> $1
    echo "GeoIP no"                                      >> $1
#    echo "GeoIPDatabase   $GEO_IP_DB"                    >> $1
    echo "IgnoreURL /get-remote-static"                  >> $1
    echo "IgnoreURL /linaro-openid/login"                >> $1
    echo "IgnoreURL /get-textile-files"                  >> $1
    echo "IgnoreURL /css/*"                              >> $1
    echo "IgnoreURL /static/*"                           >> $1
    echo "IgnoreURL /js/*"                               >> $1
    echo "IgnoreURL /license"                            >> $1

}

write_config_files ()
{
    if [ $AWFFULL -eq $TRUE ] ; then
        write.conf.file $WORK_PATH/$AWFFULL_CONF_FULL_NAME      $OUTPUT_PATH/$AWFFULL_FULL_PATH_NAME      $PROCESSED_LOG_NAME
        if [ $DEBUG -eq $TRUE ] ; then
            echo "write.awffull.conf.full"
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            write.conf.file $WORK_PATH/$AWFFULL_CONF_FILTERED_NAME  $OUTPUT_PATH/$AWFFULL_FILTERED_PATH_NAME  $FILTERED_LOG_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "write.awffull.conf.filtered"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                write.conf.file $WORK_PATH/$AWFFULL_CONF_TOOLCHAIN_NAME $OUTPUT_PATH/$AWFFULL_TOOLCHAIN_PATH_NAME $TOOLCHAIN_LOG_NAME
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "write.awffull.conf.toolchain"
                fi
            fi
        fi
    fi

    if [ $WEBALIZER -eq $TRUE ] ; then
        write.conf.file $WORK_PATH/$WEBALIZER_CONF_FULL_NAME      $OUTPUT_PATH/$WEBALIZER_FULL_PATH_NAME      $PROCESSED_LOG_NAME
        if [ $DEBUG -eq $TRUE ] ; then
            echo "write.webalizer.conf.full"
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            write.conf.file $WORK_PATH/$WEBALIZER_CONF_FILTERED_NAME  $OUTPUT_PATH/$WEBALIZER_FILTERED_PATH_NAME  $FILTERED_LOG_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "write.webalizer.conf.filtered"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                write.conf.file $WORK_PATH/$WEBALIZER_CONF_TOOLCHAIN_NAME $OUTPUT_PATH/$WEBALIZER_TOOLCHAIN_PATH_NAME $TOOLCHAIN_LOG_NAME
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "write.webalizer.conf.toolchain"
                fi
            fi
        fi
    fi

    if [ $WEBDRUID -eq $TRUE ] ; then
        write.conf.file $WORK_PATH/$WEBDRUID_CONF_FULL_NAME      $OUTPUT_PATH/$WEBDRUID_FULL_PATH_NAME      $PROCESSED_LOG_NAME
        if [ $DEBUG -eq $TRUE ] ; then
            echo "write.webdruid.conf.full"                
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            write.conf.file $WORK_PATH/$WEBDRUID_CONF_FILTERED_NAME  $OUTPUT_PATH/$WEBDRUID_FILTERED_PATH_NAME  $FILTERED_LOG_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "write.webdruid.conf.filtered"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                write.conf.file $WORK_PATH/$WEBDRUID_CONF_TOOLCHAIN_NAME $OUTPUT_PATH/$WEBDRUID_TOOLCHAIN_PATH_NAME $TOOLCHAIN_LOG_NAME
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "write.webdruid.conf.toolchain"
                fi
            fi
        fi
    fi
}

prep_directories ()
{
    if [ ! -d "$WORK_PATH" ]; then
        mkdir -p "$WORK_PATH"
        if [ $DEBUG -eq $TRUE ] ; then
            echo "mkdir -p $WORK_PATH"
        fi
    fi
    if [ $AWFFULL -eq $TRUE ] ; then
        if [ ! -d "$OUTPUT_PATH/$AWFFULL_FULL_PATH_NAME" ]; then
            mkdir -p "$OUTPUT_PATH/$AWFFULL_FULL_PATH_NAME"
            if [ $DEBUG -eq $TRUE ] ; then
                echo "mkdir -p $OUTPUT_PATH/$AWFFULL_FULL_PATH_NAME"
            fi
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            if [ ! -d "$OUTPUT_PATH/$AWFFULL_FILTERED_PATH_NAME" ]; then
                mkdir -p "$OUTPUT_PATH/$AWFFULL_FILTERED_PATH_NAME"
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "mkdir -p $OUTPUT_PATH/$AWFFULL_FILTERED_PATH_NAME"
                fi
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                if [ ! -d "$OUTPUT_PATH/$AWFFULL_TOOLCHAIN_PATH_NAME" ]; then
                    mkdir -p "$OUTPUT_PATH/$AWFFULL_TOOLCHAIN_PATH_NAME"
                    if [ $DEBUG -eq $TRUE ] ; then
                        echo "mkdir -p $OUTPUT_PATH/$AWFFULL_TOOLCHAIN_PATH_NAME"
                    fi
                fi
            fi
        fi
    fi
    if [ $WEBALIZER -eq $TRUE ] ; then
        if [ ! -d "$OUTPUT_PATH/$WEBALIZER_FULL_PATH_NAME" ]; then
            mkdir -p "$OUTPUT_PATH/$WEBALIZER_FULL_PATH_NAME"
            if [ $DEBUG -eq $TRUE ] ; then
                echo "mkdir -p $OUTPUT_PATH/$WEBALIZER_FULL_PATH_NAME"
            fi
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            if [ ! -d "$OUTPUT_PATH/$WEBALIZER_FILTERED_PATH_NAME" ]; then
                mkdir -p "$OUTPUT_PATH/$WEBALIZER_FILTERED_PATH_NAME"
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "mkdir -p $OUTPUT_PATH/$WEBALIZER_FILTER_PATH_NAME"
                fi
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                if [ ! -d "$OUTPUT_PATH/$WEBALIZER_TOOLCHAIN_PATH_NAME" ]; then
                    mkdir -p "$OUTPUT_PATH/$WEBALIZER_TOOLCHAIN_PATH_NAME"
                    if [ $DEBUG -eq $TRUE ] ; then
                       echo "mkdir -p $OUTPUT_PATH/$WEBALIZER_TOOLCHAIN_PATH_NAME"
                    fi
                fi
            fi
        fi
    fi
    if [ $WEBDRUID -eq $TRUE ] ; then
        if [ ! -d "$OUTPUT_PATH/$WEBDRUID_FULL_PATH_NAME" ]; then
            mkdir -p "$OUTPUT_PATH/$WEBDRUID_FULL_PATH_NAME"
            if [ $DEBUG -eq $TRUE ] ; then
                echo "mkdir -p $OUTPUT_PATH/$WEBDRUID_FULL_PATH_NAME"
            fi
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            if [ ! -d "$OUTPUT_PATH/$WEBDRUID_FILTERED_PATH_NAME" ]; then
                mkdir -p "$OUTPUT_PATH/$WEBDRUID_FILTERED_PATH_NAME"
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "mkdir -p $OUTPUT_PATH/$WEBDRUID_FILTERED_PATH_NAME"
                fi
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                if [ ! -d "$OUTPUT_PATH/$WEBDRUID_TOOLCHAIN_PATH_NAME" ]; then
                    mkdir -p "$OUTPUT_PATH/$WEBDRUID_TOOLCHAIN_PATH_NAME"
                    if [ $DEBUG -eq $TRUE ] ; then
                        echo "mkdir -p $OUTPUT_PATH/$WEBDRUID_TOOLCHAIN_PATH_NAME"
                    fi
                fi
            fi
        fi
    fi
    
    cd "$WORK_PATH"
}

cleanup ()
{
    # now delete the temp log file.
    if [ -f $WORK_PATH/$TMP_LOG_NAME ] ; then
        rm -f $WORK_PATH/$TMP_LOG_NAME
        if [ $DEBUG -eq $TRUE ] ; then
            echo "rm -f $WORK_PATH/$TMP_LOG_NAME"
        fi
    fi
    
    if [ -f "$WORK_PATH/$PROCESSED_LOG_NAME" ] ; then
        rm -f $WORK_PATH/$PROCESSED_LOG_NAME
        if [ $DEBUG -eq $TRUE ] ; then
            echo "rm -f $WORK_PATH/$PROCESSED_LOG_NAME"
        fi
    fi

    if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
        if [ -f "$WORK_PATH/$FILTERED_LOG_NAME" ] ; then
            rm -f $WORK_PATH/$FILTERED_LOG_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "rm -f $WORK_PATH/$FILTERED_LOG_NAME"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                rm -f $WORK_PATH/$TOOLCHAIN_LOG_NAME
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "rm -f $WORK_PATH/$TOOLCHAIN_LOG_NAME"
                fi
            fi
        fi
    fi

    if [ $AWFFULL -eq $TRUE ] ; then
        if [ -f "$WORK_PATH/$AWFFULL_CONF_FULL_NAME" ] ; then
            rm -f $WORK_PATH/$AWFFULL_CONF_FULL_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "rm -f $WORK_PATH/$AWFFULL_CONF_FULL_NAME"
            fi
        fi
    
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            if [ -f "$WORK_PATH/$AWFFULL_CONF_FILTERED_NAME" ] ; then
                rm -f $WORK_PATH/$AWFFULL_CONF_FILTERED_NAME
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "rm -f $WORK_PATH/$AWFFULL_CONF_FILTERED_NAME"
                fi
                if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                    rm -f $WORK_PATH/$AWFFULL_CONF_TOOLCHAIN_NAME
                    if [ $DEBUG -eq $TRUE ] ; then
                        echo "rm -f $WORK_PATH/$AWFFULL_CONF_TOOLCHAIN_NAME"
                    fi
                fi
            fi
        fi
    fi

    if [ $WEBALIZER -eq $TRUE ] ; then
        if [ -f "$WORK_PATH/$WEBALIZER_CONF_FULL_NAME" ] ; then
            rm -f $WORK_PATH/$WEBALIZER_CONF_FULL_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "rm -f $WORK_PATH/$WEBALIZER_CONF_FULL_NAME"
            fi
        fi
    
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            if [ -f "$WORK_PATH/$WEBALIZER_CONF_FILTERED_NAME" ] ; then
                rm -f $WORK_PATH/$WEBALIZER_CONF_FILTERED_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "rm -f $WORK_PATH/$WEBALIZER_CONF_FILTERED_NAME"
            fi
                if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                    rm -f $WORK_PATH/$WEBALIZER_CONF_TOOLCHAIN_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "rm -f $WORK_PATH/$WEBALIZER_CONF_TOOLCHAIN_NAME"
            fi
                fi
            fi
        fi
    fi

    if [ $WEBDRUID -eq $TRUE ] ; then
        if [ -f "$WORK_PATH/$WEBDRUID_CONF_FULL_NAME" ] ; then
            rm -f $WORK_PATH/$WEBDRUID_CONF_FULL_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "rm -f $WORK_PATH/$WEBDRUID_CONF_FULL_NAME"
            fi
        fi
    
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            if [ -f "$WORK_PATH/$WEBDRUID_CONF_FILTERED_NAME" ] ; then
                rm -f $WORK_PATH/$WEBDRUID_CONF_FILTERED_NAME
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "rm -f $WORK_PATH/$WEBDRUID_CONF_FILTERED_NAME"
                fi
                if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                    rm -f $WORK_PATH/$WEBDRUID_CONF_TOOLCHAIN_NAME
                    if [ $DEBUG -eq $TRUE ] ; then
                        echo "rm -f $WORK_PATH/$WEBDRUID_CONF_TOOLCHAIN_NAME"
                    fi
                fi
            fi
        fi
    fi 
}

extract_logs ()
{
    # Now in 2014 we can just preprocess all 2012 and 2013 files and save procssing time for all 3 web servers
    # then just grab all of the 2014 files to process
    # *access.log-2014*
    # preprocessed-*-2013-access.log.gz
    x=`ls $INPUT_PATH/$RAW_LOG_NAME | wc -l`
    if [ x > 0 ] ; then
       zcat $INPUT_PATH/$RAW_LOG_NAME > $WORK_PATH/$TMP_LOG_NAME
       if [ $DEBUG -eq $TRUE ] ; then
           echo "$WEB_NAME making access.log by zcat $INPUT_PATH/$RAW_LOG_NAME"
       fi
    fi
    # 2013 logs preprocessed into a single compressed file to save processing time.
    zcat $INPUT_PATH/preprocessed-*-2013-access.log.gz >  $WORK_PATH/$PROCESSED_LOG_NAME
    
}

process_logs ()
{
    ## Awffull
    # use all the data in the file
    if [ $DEBUG -eq $TRUE ] ; then
        echo "`date`"
    fi
    if [ $AWFFULL -eq $TRUE ] ; then
        awffull $AWFFULL_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$AWFFULL_CONF_FULL_NAME $WORK_PATH/$PROCESSED_LOG_NAME > /dev/null
        if [ $DEBUG -eq $TRUE ] ; then
             echo "awffull processing $WORK_PATH/$PROCESSED_LOG_NAME"
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            # Now use the filtered log that has only the .bz2 or .gz files and look at those patterns
            awffull $AWFFULL_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$AWFFULL_CONF_FILTERED_NAME $WORK_PATH/$FILTERED_LOG_NAME > /dev/null
            if [ $DEBUG -eq $TRUE ] ; then
                 echo "awffull processing $WORK_PATH/$FILTERED_LOG_NAME"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                awffull $AWFFULL_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$AWFFULL_CONF_TOOLCHAIN_NAME $WORK_PATH/$TOOLCHAIN_LOG_NAME > /dev/null
                if [ $DEBUG -eq $TRUE ] ; then
                     echo "awffull processing $WORK_PATH/$TOOLCHAIN_LOG_NAME"
                fi
            fi
        fi
    fi

    ## Webalizer
    # use all the data in the file
    if [ $DEBUG -eq $TRUE ] ; then
        echo "`date`"
    fi
    if [ $WEBALIZER -eq $TRUE ] ; then
        webalizer $WEBALIZER_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$WEBALIZER_CONF_FULL_NAME $WORK_PATH/$PROCESSED_LOG_NAME > /dev/null
        if [ $DEBUG -eq $TRUE ] ; then
            echo "webalizer processing $WORK_PATH/$PROCESSED_LOG_NAME"
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            # Now use the filtered log that has only the .bz2 or .gz files and look at those patterns
            webalizer $WEBALIZER_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$WEBALIZER_CONF_FILTERED_NAME $WORK_PATH/$FILTERED_LOG_NAME > /dev/null
            if [ $DEBUG -eq $TRUE ] ; then
                echo "webalizer processing $WORK_PATH/$FILTERED_LOG_NAME"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                webalizer $WEBALIZER_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$WEBALIZER_CONF_TOOLCHAIN_NAME $WORK_PATH/$TOOLCHAIN_LOG_NAME > /dev/null
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "webalizer processing $WORK_PATH/$TOOLCHAIN_LOG_NAME"
                fi
            fi
        fi
    fi

    ## Webdruid
    # use all the data in the file
    if [ $DEBUG -eq $TRUE ] ; then
        echo "`date`"
    fi
    if [ $WEBDRUID -eq $TRUE ] ; then
        webdruid $WEBDRUID_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$WEBDRUID_CONF_FULL_NAME $WORK_PATH/$PROCESSED_LOG_NAME > /dev/null
        if [ $DEBUG -eq $TRUE ] ; then
            echo "webdruid processing $WORK_PATH/$PROCESSED_LOG_NAME"
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            # Now use the filtered log that has only the .bz2 or .gz files and look at those patterns
            webdruid $WEBDRUID_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$WEBDRUID_CONF_FILTERED_NAME $WORK_PATH/$FILTERED_LOG_NAME > /dev/null
            if [ $DEBUG -eq $TRUE ] ; then
                echo "webdruid processing $WORK_PATH/$FILTERED_LOG_NAME"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                webdruid $WEBDRUID_EXTRA_CMD_LINE_OPTIONS -c $WORK_PATH/$WEBDRUID_CONF_TOOLCHAIN_NAME $WORK_PATH/$TOOLCHAIN_LOG_NAME > /dev/null
                if [ $DEBUG -eq $TRUE ] ; then
                    echo "webdruid processing $WORK_PATH/$TOOLCHAIN_LOG_NAME"
                fi
            fi
        fi
    fi

    ## Visitors
    # use all the data in the file
    if [ $DEBUG -eq $TRUE ] ; then
        echo "`date`"
    fi
    if [ $VISITORS -eq $TRUE ] ; then
        visitors --ignore-404 -A --prefix $WEB_NAME -o html --trails --output-file $OUTPUT_PATH/visitors-all.html $WORK_PATH/$PROCESSED_LOG_NAME &> /dev/null
        if [ $DEBUG -eq $TRUE ] ; then
            echo "visitors processing $WORK_PATH/$PROCESSED_LOG_NAME"
        fi
        if [ $EXTRACT_GZ_BZ2_FILES -eq $TRUE ] ; then
            # Now use the filtered log that has only the .bz2 or .gz files and look at those patterns
            visitors --ignore-404 -A --prefix $WEB_NAME -o html --trails --output-file $OUTPUT_PATH/visitors-filtered.html $WORK_PATH/$FILTERED_LOG_NAME &> /dev/null
            cp $WORK_PATH/$FILTERED_LOG_NAME $OUTPUT_PATH/.
            gzip -f -9 $OUTPUT_PATH/$FILTERED_LOG_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "visitors processing $WORK_PATH/$FILTERED_LOG_NAME and then creating $OUTPUT_PATH/$FILTERED_LOG_NAME.gz"
            fi
            if [ $EXTRACT_TOOLCHAIN_LOG -eq $TRUE ] ; then
                visitors --ignore-404 -A --prefix $WEB_NAME -o html --trails --output-file $OUTPUT_PATH/visitors-toolchain.html $WORK_PATH/$TOOLCHAIN_LOG_NAME &> /dev/null
                cp $WORK_PATH/$TOOLCHAIN_LOG_NAME $OUTPUT_PATH/.
                gzip -f -9 $OUTPUT_PATH/$TOOLCHAIN_LOG_NAME
            if [ $DEBUG -eq $TRUE ] ; then
                echo "visitors processing $WORK_PATH/$TOOLCHAIN_LOG_NAME and then creating $OUTPUT_PATH/$TOOLCHAIN_LOG_NAME.gz"
            fi
            fi
        fi
    fi

}

# OK, the actual section to do work, prep, write out config files as needed, 
# extract log files, process the log files, cleanup and exit.

#WEB_NAME="snapshots.linaro.org"
#WEB_NAME="releases.linaro.org"
WEB_NAME="www.linaro.org"
prep_paths_and_variables
# verify and make if necessary all directories needed by this script
prep_directories 
extract_logs

WEB_NAME="snapshots.linaro.org"
#WEB_NAME="releases.linaro.org"
#WEB_NAME="www.linaro.org"
prep_paths_and_variables
# verify and make if necessary all directories needed by this script
prep_directories 
extract_logs


#WEB_NAME="snapshots.linaro.org"
WEB_NAME="releases.linaro.org"
#WEB_NAME="www.linaro.org"
prep_paths_and_variables
# verify and make if necessary all directories needed by this script
prep_directories 
extract_logs

# done, out of here