Appendix A - PerfStat.pl

#!/usr/bin/perl -w
#
#	PerfStat.pl
#
#	Squid proxy performance statisitics (a la flexanlg)
#
#	squid log file format
#		time - UNIX UTC sec.msec
#		duration - msec
#		client - ip address
#		result code - squid type results
#		bytes - size of data delivered to client
#		request method - get,post,etc
#		URL - may contain whitespace
#		rfc931 - ident lookup for client
#		hierarchy code - source return code
#		type - content type
#
#	date		auth		mod
#	30-apr-2001	stanley wong	creation
#	29-jun-2001	stanley wong	include graphs
#	20-jul-2001	stanley wong	include type table/pie
#	03-sep-2001	stanley wong	include userid link
#	05-nov-2001 stanley wong	remove above for SysAdmin article
#
########
#
#	modules
#
use CGI qw(:standard :html3);
use POSIX qw(log10);
use GD;
use GD::Graph;
use GD::Graph::pie;
use GD::Graph::bars;
#
#	intialize counters
#
my ($start_date,$end_date) = (0,0);
#
#	initialize statistical table
#
$tstatbytmax=0;
$rstatbytmax=0;
$cstatbytmax=0;
$estatbytmax=0;
$tstattimmax=0;
$rstattimmax=0;
$cstattimmax=0;
$estattimmax=0;
$tstatbytmin=999999;
$rstatbytmin=999999;
$cstatbytmin=999999;
$estatbytmin=999999;
$tstattimmin=999999;
$rstattimmin=999999;
$cstattimmin=999999;
$estattimmin=999999;
#
#	read stdin and collect information
#
while ($inlin = <STDIN>) {
  chomp $inlin;
  @squid=(split(/\s+/,$inlin));
  $time=$squid[0];
  $duration=$squid[1];
  $client=$squid[2];
  $rtncod=$squid[3];
  $byte=$squid[4];
#  $method=$squid[5];
  $urll=$squid[6];
  $user=lc($squid[7]);
#  $pcod=$squid[8];
  $ctype=lc($squid[9]);
  my @tmp=(split(/\//,$rtncod));
  $squidrc=$tmp[0];
#  $httprc=$tmp[1];
#
#	total value counts
#
  $tstatbytc++;
  $tstatbytv += $byte;
  $tstatbyt2 += ($byte/1024)**2;
  if ($tstatbytmax < $byte) {
    $tstatbytmax = $byte;
  }
  if ($tstatbytmin > $byte) {
    $tstatbytmin = $byte;
  }
  $tstattimc++;
  $tstattimv += $duration;
  $tstattim2 += ($duration/1000)**2;
  if ($tstattimmax < $duration) {
    $tstattimmax = $duration;
  }
  if ($tstattimmin > $duration) {
    $tstattimmin = $duration;
  }
  if ($tstatbytc == 1) {
    $start_date=$time;			# start time
    $start_date =~ s/^\d+\.\d+/localtime $&/e;
  }
#
#	histogram (volume downloaded)
#
  $tvolume = int($byte/1024);
  if ($tvolume != 0) {
    $tvolmag = 10**int(log10($tvolume));
    $vbin = int($tvolume/$tvolmag)*$tvolmag;
  } else {
    $vbin = 0;
  }
  $vhist{$vbin}++;
#
#	histogram (response time)
#
  $tsecond = int($duration/1000);
  if ($tsecond != 0) {
    $tordmag = 10**int(log10($tsecond));
    $tbin = int($tsecond/$tordmag)*$tordmag;
  } else {
    $tbin = 0;
  }
  $thist{$tbin}++;
#
#	hit or miss cache counts
#
  if ($squidrc =~ /MISS/) {
    $rstatbytc++;
    $rstatbytv += $byte;
    $rstatbyt2 += ($byte/1024)**2;
    if ($rstatbytmax < $byte) {
      $rstatbytmax = $byte;
    }
    if ($rstatbytmin > $byte) {
      $rstatbytmin = $byte;
    }
    $rstattimc++;
    $rstattimv += $duration;
    $rstattim2 += ($duration/1000)**2;
    if ($rstattimmax < $duration) {
      $rstattimmax = $duration;
    }
    if ($rstattimmin > $duration) {
      $rstattimmin = $duration;
    }
  } elsif ($squidrc =~ /HIT/) {
    $cstatbytc++;
    $cstatbytv += $byte;
    $cstatbyt2 += ($byte/1024)**2;
    if ($cstatbytmax < $byte) {
      $cstatbytmax = $byte;
    }
    if ($cstatbytmin > $byte) {
      $cstatbytmin = $byte;
    }
    $cstattimc++;
    $cstattimv += $duration;
    $cstattim2 += ($duration/1000)**2;
    if ($cstattimmax < $duration) {
      $cstattimmax = $duration;
    }
    if ($cstattimmin > $duration) {
      $cstattimmin = $duration;
    }
  } else {
    $estatbytc++;
    $estatbytv += $byte;
    $estatbyt2 += ($byte/1024)**2;
    if ($estatbytmax < $byte) {
      $estatbytmax = $byte;
    }
    if ($estatbytmin > $byte) {
      $estatbytmin = $byte;
    }
    $estattimc++;
    $estattimv += $duration;
    $estattim2 += ($duration/1000)**2;
    if ($estattimmax < $duration) {
      $estattimmax = $duration;
    }
    if ($estattimmin > $duration) {
      $estattimmin = $duration;
    }
  }
#
#	url counts
#
  $urls = $urll;
  $urls =~ s#(^.*?//)(.*?/).*$#$2#;
  $urls =~ s#/$##;
  $urls =~ s#:.*$##;
  $site{$urls}++;
  $sit_byte{$urls} += $byte;
  $sit_time{$urls} += $duration;
#
#	country counts
#
  $country = $urls;
  $country =~ s/.*\.//;
  if ($country =~ m/\d/) {
    $country="UN";
  }
  $_ = $country;
  $ccount = tr/a-z//;
  if ($ccount == 3) {
    $country="us";
  } elsif ($ccount > 3) {
    $country="UN";
  }
  $sit_co{$country}++;
  $sit_cbyt{$country} += $byte;
  $sit_ctim{$country} += $duration;
#
#	top level domain counts
#
  $domain = $urls;
  if ($country =~ m/us/) {
    $domain =~ s/.*\.//;
  } elsif ($country =~ m/UN/) {
    $domain = $country;
  } else {
    $domain =~ s/\...$//;
    $domain =~ s/.*\.//;
  }
  $sit_dom{$domain}++;
  $sit_dbyt{$domain} += $byte;
  $sit_dtim{$domain} += $duration;
#
#	client counts
#
  $client_host{$client}++;
  $client_byte{$client} += $byte;
  $client_time{$client} += $duration;
#
#	user counts
#
  $user_client{$user}++;
  $user_byte{$user} += $byte;
  $user_time{$user} += $duration;
#
#	content type counts
#
  $contyp_cnt{$ctype}++;
  $contyp_byt{$ctype} += $byte;
  $contyp_tim{$ctype} += $duration;
#
#	end date
#
  $end_date=$time;				# end time
}  
$end_date =~ s/^\d+\.\d+/localtime $&/e;
#
#	end of data collection
#
############################################################
#
#	output html code
#
print <<EOP;
<html>
<head>
<title>cache proxy statistics</title>
<link rel=stylesheet href="/proxy/style.css" type="text/css">
</head>
<body> 
<h2>gw-proxy performance statistics</h2>
<p></p>
<p><smaller><strong>For the period between $start_date and $end_date
</strong></smaller></p>
<hr align=center>
<p></p>
<center>
EOP
#
#	general results
#
print("<table border=0>\n");
print("  <tr>\n");
print("    <td align=\"right\">Total Access:</td>\n");
printf("    <td align=\"right\">%12d</td></tr>\n",$tstatbytc);
print("  <tr>\n");
print("    <td align=\"right\">Total Kbytes:</td>\n");
printf("    <td align=\"right\">%12d</td></tr>\n",($tstatbytv/1024));
print("  <tr>\n");
print("    <td align=\"right\">Total Duration (sec):</td>\n");
printf("    <td align=\"right\">%12d</td></tr>\n",($tstattimv/1000));
@nsit=keys(%site);
print("  <tr>\n");
print("    <td align=\"right\">Unique URL's:</td>\n");
printf("    <td align=\"right\">%12d</td></tr>\n",($#nsit+1));
@nclient=keys(%client_host);
print("  <tr>\n");
print("    <td align=\"right\">Unique client's:</td>\n");
printf("    <td align=\"right\">%12d</td></tr>\n",($#nclient+1));
@nuser=keys(%user_client);
print("  <tr>\n");
print("    <td align=\"right\">Unique users's:</td>\n");
printf("    <td align=\"right\">%12d</td></tr>\n",($#nuser+1));
print ("</table>\n");
print ("<p></p>\n");
#
#	cache performance statistics
#
print("<h3>Cache Performance Statistics</h3>\n");
#
#	counts
#
print("<table border=2 width=400>\n");
print("  <tr>\n");
print("    <th></th>\n");
print("    <th align=center colspan=2>Access</th>\n");
print("    <th align=center colspan=2>Kbyte</th>\n");
print("    <th align=center colspan=2>Sec</th></tr>\n");
print("  <tr>\n");
print("    <td>Remote</td>\n");
printf("    <td align=\"right\">%12d</td>\n",
	$rstattimc);
printf("    <td align=\"right\">%4.1f%s</td>\n",
	($rstattimc/$tstattimc)*100,chr(37));
printf("    <td align=\"right\">%12d</td>\n",
	($rstatbytv/1024));
printf("    <td align=\"right\">%4.1f%s</td>\n",
	($rstatbytv/$tstatbytv)*100,chr(37));
printf("    <td align=\"right\">%12d</td>\n",
	($rstattimv/1000));
printf("    <td align=\"right\">%4.1f%s</td></tr>\n",
	($rstattimv/$tstattimv)*100,chr(37));
print("  <tr>\n");
print("    <td>Cache</td>\n");
printf("    <td align=\"right\">%12d</td>\n",
	$cstattimc);
printf("    <td align=\"right\">%4.1f%s</td>\n",
	($cstattimc/$tstattimc)*100,chr(37));
printf("    <td align=\"right\">%12d</td>\n",
	($cstatbytv/1024));
printf("    <td align=\"right\">%4.1f%s</td>\n",
	($cstatbytv/$tstatbytv)*100,chr(37));
printf("    <td align=\"right\">%12d</td>\n",
	($cstattimv/1000));
printf("    <td align=\"right\">%4.1f%s</td></tr>\n",
	($cstattimv/$tstattimv)*100,chr(37));
print("  <tr>\n");
print("    <td>Denied</td>\n");
printf("    <td align=\"right\">%12d</td>\n",
	$estattimc);
printf("    <td align=\"right\">%4.1f%s</td>\n",
	($estattimc/$tstattimc)*100,chr(37));
printf("    <td align=\"right\">%12d</td>\n",
	($estatbytv/1024));
printf("    <td align=\"right\">%4.1f%s</td>\n",
	($estatbytv/$tstatbytv)*100,chr(37));
printf("    <td align=\"right\">%12d</td>\n",
	($estattimv/1000));
printf("    <td align=\"right\">%4.1f%s</td></tr>\n",
	($estattimv/$tstattimv)*100,chr(37));
print("  <tr>\n");
print("    <td>TOTAL</td>\n");
printf("    <td align=\"right\">%12d</td>\n",$tstattimc);
printf("    <td align=\"right\">%4.1f%s</td>\n","100",chr(37));
printf("    <td align=\"right\">%12d</td>\n",($tstatbytv/1024));
printf("    <td align=\"right\">%4.1f%s</td>\n","100",chr(37));
printf("    <td align=\"right\">%12d</td>\n",($tstattimv/1000));
printf("    <td align=\"right\">%4.1f%s</td></tr>\n","100",chr(37));
print("</table>\n");
print("<p></p>\n");
#
#	pie charts
#
print("<table border=0 width=540>\n");
print("  <tr>\n");
print("    <td align=center><img src=\"PApie.png\"></td>\n");
print("    <td align=center><img src=\"PVpie.png\"></td>\n");
print("  </tr>\n");
print("</table>\n");
print("<p></p>\n");
#
#	Kbyte mean, std-dev, mode, min, max
#
print<<EOP;
<table border=2 width=400>
  <tr>
    <th></th>
    <th align=center>Kbyte/req</th>
    <th align=center>std dev</th>
    <th align=center>median</th>
    <th align=center>min</th>
    <th align=center>max</th></tr>
EOP
#
print("  <tr>\n");
print("    <td>Remote</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$rstatbytv/$rstatbytc/1024);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($rstatbyt2-(($rstatbytv/1024)**2/$rstatbytc))/$rstatbytc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($rstatbytmax-$rstatbytmin)/1024/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$rstatbytmin/1024);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$rstatbytmax/1024);
print("  <tr>\n");
print("    <td>Cache</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$cstatbytv/$cstatbytc/1024);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($cstatbyt2-(($cstatbytv/1024)**2/$cstatbytc))/$cstatbytc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($cstatbytmax-$cstatbytmin)/1024/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$cstatbytmin/1024);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$cstatbytmax/1024);
print("  <tr>\n");
print("    <td>Denied</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$estatbytv/$estatbytc/1024);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($estatbyt2-(($estatbytv/1024)**2/$estatbytc))/$estatbytc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($estatbytmax-$tstatbytmin)/1024/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$estatbytmin/1024);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$estatbytmax/1024);
print("  <tr>\n");
print("    <td>TOTAL</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$tstatbytv/$tstatbytc/1024);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($tstatbyt2-(($tstatbytv/1024)**2/$tstatbytc))/$tstatbytc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($tstatbytmax-$tstatbytmin)/1024/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$tstatbytmin/1024);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$tstatbytmax/1024);
print("</table>\n");
#
#	volume download histogram
#
print("<p><img src=\"VDhist.png\"></p>\n");
#
#	Sec mean, std-dev, mode, min, max
#
print<<EOP;
<table border=2 width=400>
  <tr>
    <th></th>
    <th align=center>Sec/req</th>
    <th align=center>std dev</th>
    <th align=center>median</th>
    <th align=center>min</th>
    <th align=center>max</th></tr>
EOP
#
print("  <tr>\n");
print("    <td>Remote</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$rstattimv/$rstattimc/1000);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($rstattim2-(($rstattimv/1024)**2/$rstattimc))/$rstattimc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($rstattimmax-$rstattimmin)/1000/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$rstattimmin/1000);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$rstattimmax/1000);
print("  <tr>\n");
print("    <td>Cache</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$cstattimv/$cstattimc/1000);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($cstattim2-(($cstattimv/1024)**2/$cstattimc))/$cstattimc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($cstattimmax-$cstattimmin)/1000/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$cstattimmin/1000);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$cstattimmax/1000);
print("  <tr>\n");
print("    <td>Denied</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$estattimv/$estattimc/1000);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($estattim2-(($estattimv/1024)**2/$estattimc))/$estattimc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($estattimmax-$estattimmin)/1000/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$estattimmin/1000);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$estattimmax/1000);
print("  <tr>\n");
print("    <td>TOTAL</td>\n");
printf("    <td align=\"right\">%10.3f</td>\n",
	$tstattimv/$tstattimc/1000);
printf("    <td align=\"right\">%10.3f</td>\n",
	sqrt(($tstattim2-(($tstattimv/1024)**2/$tstattimc))/$tstattimc));
printf("    <td align=\"right\">%10.3f</td>\n",
	($tstattimmax-$tstattimmin)/1000/2);
printf("    <td align=\"right\">%10.3f</td>\n",
	$tstattimmin/1000);
printf("    <td align=\"right\">%10.3f</td></tr>\n",
	$tstattimmax/1000);
print("  <tr>\n");
print("    <th>Improvement</th>\n");
printf("    <th align=\"right\">%10.3f</th></tr>\n",
	(($rstattimv/$rstattimc)-($tstattimv/$tstattimc))/1000);
print("</table>\n");
#
#	response time histogram
#
print("<p><img src=\"RThist.png\"></p>\n");
#
#	top 20 sites
#
printf("<h3>Top 20 Sites (out of %6d)</h3>\n",($#nsit+1));
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Site</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
my $tcount=0;
foreach $item (sort {$site{$b} <=> $site{$a}} keys %site) {
  $tcount++;
  last if ($tcount > 20);
  if ($tcount <= 20) {
    print("  <tr>\n");
    printf("    <td>%30s</td>\n",$item);
    printf("    <td align=\"right\">%12d</td>\n",
	$site{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($site{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_byte{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_byte{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$sit_byte{$item}/$site{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_time{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_time{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$sit_time{$item}/$site{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$sit_time{$item}/($sit_byte{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
#
#	top 10 countries
#
my $ccfile="/opt/local/squid/etc/CountryCodes.txt";
open(CCFIL,$ccfile) ||
	die "ERROR: cannot read $ccfile: $!\n";
while(<CCFIL>) {
  chomp;
  @ccrow=split(/\s+/,$_,2);
  $ccd=$ccrow[0];
  $cntry=$ccrow[1];
  $ccode{$ccd}=$cntry;
}
close(CCFIL);
#
print("<h3>Top 10 Countries</h3>\n");
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Country</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
$tcount=0;
foreach $item (sort {$sit_co{$b} <=> $sit_co{$a}} keys %sit_co) {
  $tcount++;
  last if ($tcount > 10);
  if ($tcount <= 10 && $ccode{$item}) {
    print("  <tr>\n");
    printf("    <td><strong>%3s</strong>%25s</td>\n",$item,$ccode{$item});
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_co{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_co{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_cbyt{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_cbyt{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$sit_cbyt{$item}/$sit_co{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_ctim{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_ctim{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$sit_ctim{$item}/$sit_co{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$sit_ctim{$item}/($sit_cbyt{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
#
#	pie charts
#
print("<table border=0 width=540>\n");
print("  <tr>\n");
print("    <td align=center><img src=\"CHpie.png\"></td>\n");
print("    <td align=center><img src=\"CVpie.png\"></td>\n");
print("  </tr>\n");
print("</table>\n");
print("<p></p>\n");
#
#	top 10 domains
#
my $domfile="/opt/local/squid/etc/Domain.txt";
open(DOMFIL,$domfile) ||
	die "ERROR: cannot read $domfile: $!\n";
while(<DOMFIL>) {
  chomp;
  @domrow=split(/\s+/,$_,2);
  $domcd=$domrow[0];
  $domtp=$domrow[1];
  $dom{$domcd}=$domtp;
}
close(DOMFIL);
#
print("<h3>Top 10 Domains</h3>\n");
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Domain</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
$tcount=0;
foreach $item (sort {$sit_dom{$b} <=> $sit_dom{$a}} keys %sit_dom) {
  $tcount++;
  last if ($tcount > 10);
  if ($tcount <= 10 && $sit_dom{$item}) {
    print("  <tr>\n");
    if ($item =~ m/\d/) {
      printf("    <td><strong>%6s</strong>%25s</td>\n",
	$item,"Unknown");
    } elsif ($dom{$item}) {
      printf("    <td><strong>%6s</strong>%25s</td>\n",
	$item,$dom{$item});
    } else {
      printf("    <td><strong>%6s</strong>%25s</td>\n",
	$item," ");
    }
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_dom{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_dom{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_dbyt{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_dbyt{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$sit_dbyt{$item}/$sit_dom{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$sit_dtim{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($sit_dtim{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$sit_dtim{$item}/$sit_dom{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$sit_dtim{$item}/($sit_dbyt{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
#
#	pie charts
#
print("<table border=0 width=540>\n");
print("  <tr>\n");
print("    <td align=center><img src=\"DHpie.png\"></td>\n");
print("    <td align=center><img src=\"DVpie.png\"></td>\n");
print("  </tr>\n");
print("</table>\n");
print("<p></p>\n");
#
#	top 10 content types
#
print("<h3>Top 10 Content types</h3>\n");
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Content</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
$tcount=0;
foreach $item (sort {$contyp_cnt{$b} <=> $contyp_cnt{$a}} keys %contyp_cnt) {
  $tcount++;
  if ($tcount <= 10 && $contyp_cnt{$item}) {
    print("  <tr>\n");
    printf("    <td>%30s</td>\n",
	$item);
    printf("    <td align=\"right\">%12d</td>\n",
	$contyp_cnt{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($contyp_cnt{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$contyp_byt{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($contyp_byt{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$contyp_byt{$item}/$contyp_cnt{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$contyp_tim{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($contyp_tim{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$contyp_tim{$item}/$contyp_cnt{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$contyp_tim{$item}/($contyp_byt{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
#
#	pie charts
#
print("<table border=0 width=540>\n");
print("  <tr>\n");
print("    <td align=center><img src=\"THpie.png\"></td>\n");
print("    <td align=center><img src=\"TVpie.png\"></td>\n");
print("  </tr>\n");
print("</table>\n");
print("<p></p>\n");
#
#	top 20 clients
#
printf("<h3>Top 20 Clients (out of %6d)</h3>\n",($#nclient+1));
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Client</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
$tcount=0;
foreach $item (sort {$client_host{$b} <=> $client_host{$a}} keys %client_host) {
  $tcount++;
  last if ($tcount > 20);
  if ($tcount <= 20 && $client_host{$item}) {
    print("  <tr>\n");
    printf("    <td>%20s</td>\n",$item);
    printf("    <td align=\"right\">%12d</td>\n",
	$client_host{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($client_host{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$client_byte{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($client_byte{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$client_byte{$item}/$client_host{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$client_time{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($client_time{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$client_time{$item}/$client_host{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$client_time{$item}/($client_byte{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
#
#	top 20 users (access)
#
printf("<h3>Top 20 Users by access (out of %6d)</h3>\n",($#nuser+1));
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Users</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
$tcount=0;
foreach $item (sort {$user_client{$b} <=> $user_client{$a}} keys %user_client) {
  $tcount++;
  last if ($tcount > 20);
  if ($tcount <= 20 && $user_client{$item}) {
    print("  <tr>\n");
    printf("    <td>%s</td>\n",
	$item,$item);
    printf("    <td align=\"right\">%12d</td>\n",
	$user_client{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_client{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$user_byte{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_byte{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$user_byte{$item}/$user_client{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$user_time{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_time{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$user_time{$item}/$user_client{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$user_time{$item}/($user_byte{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
#
#	top 20 users (volume)
#
printf("<h3>Top 20 Users by volume (out of %6d)</h3>\n",($#nuser+1));
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Users</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
$tcount=0;
foreach $item (sort {$user_byte{$b} <=> $user_byte{$a}} keys %user_byte) {
  $tcount++;
  last if ($tcount > 20);
  if ($tcount <= 20 && $user_byte{$item}) {
    print("  <tr>\n");
    printf("    <td>%s</td>\n",
	$item,$item);
    printf("    <td align=\"right\">%12d</td>\n",
	$user_client{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_client{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$user_byte{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_byte{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$user_byte{$item}/$user_client{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$user_time{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_time{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$user_time{$item}/$user_client{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$user_time{$item}/($user_byte{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
#
#	top 20 users (time)
#
printf("<h3>Top 20 Users by time (out of %6d)</h3>\n",($#nuser+1));
#
print<<EOP;
<table border=2 width=550>
  <tr>
    <th align=center>Users</th>
    <th align=center colspan=2>Access</th>
    <th align=center colspan=2>Kbyte</th>
    <th align=center>Kb/req</th>
    <th align=center colspan=2>Sec</th>
    <th align=center>Sec/req</th>
    <th align=center>Sec/Kb</th></tr>
EOP
#
$tcount=0;
foreach $item (sort {$user_time{$b} <=> $user_time{$a}} keys %user_time) {
  $tcount++;
  last if ($tcount > 20);
  if ($tcount <= 20 && $user_time{$item}) {
    print("  <tr>\n");
    printf("    <td>%s</td>\n",
	$item,$item);
    printf("    <td align=\"right\">%12d</td>\n",
	$user_client{$item});
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_client{$item}/$tstatbytc*100),chr(37));
    printf("    <td align=\"right\">%12d</td>\n",
	$user_byte{$item}/1024);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_byte{$item}/$tstatbytv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$user_byte{$item}/$user_client{$item}/1024);
    printf("    <td align=\"right\">%12d</td>\n",
	$user_time{$item}/1000);
    printf("    <td align=\"right\">%4.1f%s</td>\n",
	($user_time{$item}/$tstattimv*100),chr(37));
    printf("    <td align=\"right\">%8.3f</td>\n",
	$user_time{$item}/$user_client{$item}/1000);
    printf("    <td align=\"right\">%8.3f</td></tr>\n",
	$user_time{$item}/($user_byte{$item}/1024)/1000);
  }
}
print("</table>\n");
print("<p></p>\n");
print("</center>\n");
#
print("<hr align=center>\n");
#
$now=`date`;
printf("<p>PerfStat generated at %21s</p>\n",$now);
print("</body>\n</html>\n");
#
#	end of html output
#
############################################################
#
#	access pie chart
#
@pdata = (
  [ "Remote", "Cache", "Denied" ],
  [ $rstattimc, $cstattimc, $estattimc ]
);
$apie = GD::Graph::pie->new(250,260);
$apie->set(
  title		=> "Web Cache Proxy Hits",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);
my $pagd = $apie->plot(\@pdata);
open(IMG,">PApie.png") or die $!;
binmode IMG;
print IMG $pagd->png;
close IMG;
#
#	volume pie chart
#
@pdata = (
  [ "Remote", "Cache", "Denied" ],
  [ $rstatbytv, $cstatbytv, $estatbytv ]
);
$vpie = GD::Graph::pie->new(250,260);
$vpie->set(
  title		=> "Proxy Hits (Volume)",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);    
my $pvgd = $vpie->plot(\@pdata);
open(IMG,">PVpie.png") or die $!;
binmode IMG;
print IMG $pvgd->png;
close IMG;
#
#	volume download histogram
#
$vhmax = 0;
foreach $key (sort {$a <=> $b} keys %vhist) {
  $val = $vhist{$key};
  if ($val > $vhmax) {
    $vhmax = $val;
  }
  $vhkey = $key;
}
#
@xdata=();
@ydata=();
$bin_sz=1;
for ($key = 0; $key <= $vhkey; $key += $bin_sz) {
  if ($vhist{$key}) {
    $val = $vhist{$key};
  } else {
    $val = 0;
  }
  if ($key != 0) {
    $bin_sz=10**(int(log10($key)));
  }
  $key =~ s/000$/k/;
  push @xdata, $key;
  push @ydata, $val;
  $key =~ s/k/000/;
}
#
$real_max = $vhmax;
&nice_scale;
#
@hdata = ( [ @xdata ], [ @ydata ] );
$histogram = GD::Graph::bars->new(600,400);
$histogram->set(
  x_label_position	=> 1/2,
  x_label	=> "Volume Download (kbyte)",
  y_label	=> "Accesses",
  title		=> "Proxy Volume Download histogram",
  y_max_value	=> $nice_max,
  y_tick_number	=> $factor,
  bgclr		=> "lgray",
  t_margin	=> 10,
  b_margin	=> 5,
  l_margin	=> 5,
  r_margin	=> 5,
);
my $kgd = $histogram->plot(\@hdata);
open(IMG,'>VDhist.png') or die $!;
binmode IMG;
print IMG $kgd->png;
close IMG;
#
#	response time histogram
#
$thmax = 0;
foreach $key (sort {$a <=> $b} keys %thist) {
  $val = $thist{$key};
  if ($val > $thmax) {
    $thmax = $val;
  }
  $thkey = $key;
}
#
@xdata=();
@ydata=();
$bin_sz=1;
for ($key = 0; $key <= $thkey; $key += $bin_sz) {
  if ($thist{$key}) {
    $val = $thist{$key};
  } else {
    $val = 0;
  }
  if ($key != 0) {
    $bin_sz=10**(int(log10($key)));
  }
  $key =~ s/000$/k/;
  push @xdata, $key;
  push @ydata, $val;
  $key =~ s/k/000/;
}
#
$real_max = $thmax;
&nice_scale;
#
@hdata = ( [ @xdata ], [ @ydata ] );
$histogram = GD::Graph::bars->new(600,400);
$histogram->set(
  x_label_position	=> 1/2,
  x_label	=> "Response Time (seconds)",
  y_label	=> "Accesses",
  title		=> "Proxy Response Time histogram",
  y_max_value	=> $nice_max,
  y_tick_number	=> $factor,
  bgclr		=> "lgray",
  t_margin	=> 10,
  b_margin	=> 5,
  l_margin	=> 5,
  r_margin	=> 5,
);
my $hgd = $histogram->plot(\@hdata);
open(IMG,'>RThist.png') or die $!;
binmode IMG;
print IMG $hgd->png;
close IMG;
#
#	country pie chart
#
@cname=();
$tcount=0;
foreach $item (sort {$sit_co{$b} <=> $sit_co{$a}} keys %sit_co) {
  $tcount++;
  if ($tcount <= 10 && $sit_co{$item}) {
    push @cname, $item;
    push @caccs, $sit_co{$item};
  }
}
#
@cdata = ( [ @cname ], [ @caccs ] );
$cpie = GD::Graph::pie->new(250,260);
$cpie->set(
  title		=> "Top 10 Countries (accesses)",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);
my $chgd = $cpie->plot(\@cdata);
open(IMG,">CHpie.png") or die $!;
binmode IMG;
print IMG $chgd->png;
close IMG;
#
@cname=();
$tcount=0;
foreach $item (sort {$sit_cbyt{$b} <=> $sit_cbyt{$a}} keys %sit_cbyt) {
  $tcount++;
  if ($tcount <= 10 && $sit_cbyt{$item}) {
    push @cname, $item;
    push @cbyte, $sit_cbyt{$item};
  }
}
#
@cdata = ( [ @cname ], [ @cbyte ] );
$cpie = GD::Graph::pie->new(250,260);
$cpie->set(
  title		=> "Top 10 Countries (volume)",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);    
my $cvgd = $cpie->plot(\@cdata);
open(IMG,">CVpie.png") or die $!;
binmode IMG;
print IMG $cvgd->png;
close IMG;
#
#	domain pie chart
#
@dname=();
$tcount=0;
foreach $item (sort {$sit_dom{$b} <=> $sit_dom{$a}} keys %sit_dom) {
  $tcount++;
  last if ($tcount > 10);
  if ($tcount <= 10 && $sit_dom{$item}) {
    push @dname, $item;
    push @daccs, $sit_dom{$item};
  }
}
#
@ddata = ( [ @dname ], [ @daccs ] );
$dpie = GD::Graph::pie->new(250,260);
$dpie->set(
  title		=> "Top 10 Domains (accesses)",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);
my $dhgd = $dpie->plot(\@ddata);
open(IMG,">DHpie.png") or die $!;
binmode IMG;
print IMG $dhgd->png;
close IMG;
#
@dname=();
$tcount=0;
foreach $item (sort {$sit_dbyt{$b} <=> $sit_dbyt{$a}} keys %sit_dbyt) {
  $tcount++;
  last if ($tcount > 10);
  if ($tcount <= 10 && $sit_dbyt{$item}) {
    push @dname, $item;
    push @dbyte, $sit_dbyt{$item};
  }
}
#
@ddata = ( [ @dname ], [ @dbyte ] );
$dpie = GD::Graph::pie->new(250,260);
$dpie->set(
  title		=> "Top 10 Domains (volume)",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);    
my $dvgd = $dpie->plot(\@ddata);
open(IMG,">DVpie.png") or die $!;
binmode IMG;
print IMG $dvgd->png;
close IMG;
#
#	content type pie chart
#
@tname=();
$tcount=0;
foreach $item (sort {$contyp_cnt{$b} <=> $contyp_cnt{$a}} keys %contyp_cnt) {
  $tcount++;
  last if ($tcount > 10);
  if ($tcount <= 10 && $contyp_cnt{$item}) {
    push @tname, $item;
    push @taccs, $contyp_cnt{$item};
  }
}
#
@tdata = ( [ @tname ], [ @taccs ] );
$tpie = GD::Graph::pie->new(250,260);
$tpie->set(
  title		=> "Top 10 Content Types (accesses)",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);
my $thgd = $tpie->plot(\@tdata);
open(IMG,">THpie.png") or die $!;
binmode IMG;
print IMG $thgd->png;
close IMG;
#
@tname=();
$tcount=0;
foreach $item (sort {$contyp_byt{$b} <=> $contyp_byt{$a}} keys %contyp_byt) {
  $tcount++;
  last if ($tcount > 10);
  if ($tcount <= 10 && $contyp_byt{$item}) {
    push @tname, $item;
    push @tbyte, $contyp_byt{$item};
  }
}
#
@tdata = ( [ @tname ], [ @tbyte ] );
$tpie = GD::Graph::pie->new(250,260);
$tpie->set(
  title		=> "Top 10 Content Types (volume)",
  bgclr		=> "lgray",
  suppress_angle	=> 4,
);    
my $tvgd = $tpie->plot(\@tdata);
open(IMG,">TVpie.png") or die $!;
binmode IMG;
print IMG $tvgd->png;
close IMG;
#
#	nice scale function
#
sub nice_scale() {
  if ($real_max <= 0) {
    print "error: max value must be greater than $real_max\n";
    exit(0);
  }
#
#	calc order of magnitude
#
  $ordmag = 10**(int(log10($real_max)));
  $facmag = $real_max/$ordmag;
  if ($facmag < 2) {
    $scale = $ordmag/5;
  } else {
    $scale = $ordmag/2;
  }
  &factor;
  while ($factor < 4) {
    $scale = $scale/2;
    &factor;
  }
  while ($factor >= 8) {
    $scale = $scale*2;
    &factor;
  }
$nice_max = $factor*$scale;
}
#
#	factor function
#
sub factor() {
  $remain = ($real_max/$scale)-int($real_max/$scale);
  if ($remain > 0) {
    $factor = int($real_max/$scale)+1;
  } else {
    $factor = int($real_max/$scale);
  }
}
#
#	end of plots
#
############################################################



 Appendix B - Squid Report cron shell script example
#!/bin/sh
#
#	SquidReport.sh
#
#	Example shell script to perform analysis on squid log files
#	and rename files for current year and month.  Assuming all
#	the logs are kept in one file.  In reality, there would be
#	several weekly or daily log files, possibly compressed.
#
SQDIR=/opt/local/squid
SQLOG=$SQDIR/logs
SQWEB=$SQDIR/htdocs
SQBIN=$SQDIR/bin
#
SQACC=$SQLOG/access.log
SQPRM=$SQBIN/PerfStat.pl
SQRPT=$SQWEB/perf.html
#
SQDAT=`date -u "%Y.%m"`
#
#	change graphic file names function
#
fix_graphics()
{
  for i in CHpie CVpie DHpie DVpie PApie PVpie THpie TVpie RThist VDhist
  do
    if [ -s $SQWEB/${i}.png ]; then
      mv $SQWEB/${i}.png $SQWEB/${i}_${SQDAT}.png
    else
      echo "error: image file [$i] missing or empty"
    fi
  done
  if [ -s $SQRPT ]; then
    sed "s/\.png/_${SQDAT}\.png/" < $SQRPT > ${SQRPT%\.html}_${SQDAT}.html
    rm $SQRPT
  else
    echo "error: report file [$SQRPT] missing or empty"
  fi
}
#
#	main
#
cd $SQWEB
cat $SQACC | $SQPRM > $SQRPT && fix_graphics
#

