;This WinBatch script - with some luck - will process
;standard NCSA Log files. It will not process MSII files.
;It should not be hard to do, I just have not done it it
;This script takes a LOG file and makes a HTML file summary,
;suitable for posting.
; For BIG log files....Use FAST machines. This is a real
; number cruncher type script using advanced WinBatch features.
; It has been optimized for speed. However WinBatch was not really
; designed for this task, so special-purpose log analyzers will
; of course beat it hands down. But then this works, and it can
; quietly run in the background while you do other work.
logfile=AskFileName("Web Report Writer","","*.log|Log Files|*.*|All Files|","*.log",1)
reportfn=strcat(FilePath(logfile),FileRoot(logfile),".HTML")
;-------------------------------------------------
BoxOpen("Processing %logfile% -> %reportfn%","Initializing")
;pause(logfile,reportfn)
;WEBBATCH NCSA Script reporter
param1=0
param2=0
param3=0
param4=0
param5=0
param6=0
param7=0
param8=0
param9=0
paramA=0
paramB=0
paramC=0
x=@string
x=@ascending
begin=GetExactTime()
Exclusive(@on)
ttt0=GetTickCount()
tstamp=GetExactTime()
cmc=20000
cns=96
compcountersize=4
cts=cns+compcountersize
compalloc=cmc*cts
cb=BinaryAlloc(compalloc+cts)
BinaryEODSet(cb,compalloc+cts)
cno=0
filemaxcount=10000
fns=196
filecountersize=4
fts=fns+filecountersize
filealloc=filemaxcount*fts
fb=BinaryAlloc(filealloc+fts)
BinaryEODSet(fb,filealloc+fts)
fno=0
datemaxcount=400
datenamesize=12
datecountersize=4
datetotalsize=datenamesize+datecountersize
datealloc=datemaxcount*datetotalsize
db=BinaryAlloc(datealloc+datetotalsize)
datenameoffset=0
dash="XXX"
dindex=-1
for x=0 to 31
day%x%=0
t%x%=0
next
error404=0
error500=0
tb=0
logsize=FileSize(logfile)+2000
log=BinaryAlloc(logsize)
logsize=BinaryRead(log,logfile)
;Tricky part here. Take an educated guess at the line delimiter.
; this code should pick out LF, CR or CRLF
; It will get fooled and fail on LFCR or other combos
crcnt=BinaryStrCnt(log,0,min(logsize-1,5000),@CR)
lfcnt=BinaryStrCnt(log,0,min(logsize-1,5000),@LF)
if crcnt==0 && lfcnt!=0
delim=@LF
else
if crcnt!=0 && lfcnt==0
delim=@cr
else
if abs(crcnt-lfcnt)<3
delim=@CRLF
else
Message("Ooops","Cannot determine line delimiters on file")
exit
endif
endif
endif
delimsize=strlen(delim)
logindex=0
logcount=0
lc=0
ttt1=GetTickCount()
while 1
newIndex=BinaryIndex(log,logindex,delim,@fwdscan)
if newindex==0 then break
line=BinaryPeekStr(log,logindex,newindex-logindex)
logindex=newindex+delimsize
ParseData(line)
;---
cname=param1
xx=strreplace(param4,'[','')
dadate=ItemExtract(1,xx,':')
dayom=ItemExtract(1,dadate,"/")+0
y=strlen(dadate)
datime=strsub(xx,y+2,-1)
rsltcode=param7
if param8=="-" then numbytes=0
else numbytes=param8
dafn=strlower(ItemExtract(2,param6,' '))
if strindex("?",dafn,0,@fwdscan) then dafn=strcat(ItemExtract(1,dafn,'?')," ")
;---
if !lc then gosub dafirst
if rsltcode==404 || rsltcode==500 then gosub daerror
lc=lc+1
BoxText("Processing line %lc%")
tb=tb+numbytes
o=BinaryHashRec(cb,cts,cno,cns,cname)
BinaryIncr4(cb,o+cns)
o=BinaryHashRec(fb,fts,fno,fns,dafn)
BinaryIncr4(fb,o+fns)
if dash!=dadate then dindex=dindex+1
then dash=dadate
o=dindex*datetotalsize
if BinaryPeek(db,o)==0 then BinaryPokeStr(db,o,dadate)
BinaryIncr4(db, o+datenamesize)
day%dayom%=day%dayom%+1
tx=strsub(datime,1,2)
tx=tx+0
t%tx%=t%tx%+1
end while
BoxText("Building report headers")
ttt2=GetTickCount()
lastd=dadate
lastt=datime
;BinaryWrite(cb,"comp.before")
;BinaryWrite(fb,"file.before")
BinarySort(cb,cts,cno,cns,@STRING|@ASCENDING)
;BinaryWrite(cb,"comp.after1")
;BinaryWrite(fb,"file.after1")
BinarySort(fb,fts,fno,fns,@STRING|@ASCENDING)
;BinaryWrite(cb,"comp.after2")
;BinaryWrite(fb,"file.after2")
;Also @DECENDING @WORD1 @WORD2 @WORD4 @FLOAT8
ttt3=GetTickCount()
;Display(5,'Processing','Complete')
;boxText("Writing HTML Headers")
rep=FileOpen(reportfn,'WRITE')
FileWrite(rep,'<HTML><HEAD><TITLE>WebBatch Statistics Report</TITLE></HEAD><BODY>')
FileWrite(rep,'<h1>WebBatch Statistics Report</h1>')
FileWrite(rep,strcat('<em> Last Updated: ',TimeDate(),'</em>'))
FileWrite(rep,'<ul>')
FileWrite(rep,'<LI><A HREF="#Daily">Daily Statistics</A>')
FileWrite(rep,'<LI><A HREF="#Hourly">Hourly Statistics</A>')
FileWrite(rep,'<LI><A HREF="#Dest">Total by Destination</A>')
FileWrite(rep,'<LI><A HREF="#Archive">Total by File</A>')
FileWrite(rep,'</ul>')
FileWrite(rep,'<h2>Total for Summary Period</h2><b>%firstd%</b> %firstt%<br><blockquote>to</blockquote><b>%lastd%</b> %lastt%<br><pre>')
FileWrite(rep,'Files Transmitted %lc%')
FileWrite(rep,'Bytes Transmitted %tb%')
FileWrite(rep,'File not found errors %error404%')
FileWrite(rep,'CGI Errors %error500%')
FileWrite(rep,'</pre><hr><h2><A NAME="Daily">Daily Statistics</A></H2><pre>')
;boxText("Writing Summaries by Date")
FileWrite(rep,'Date Files Sent')
FileWrite(rep,'------------------------------')
;datecount=0
;for offset= 0 to datealloc by datetotalsize
; if BinaryPeek(db,offset)==0 then continue
; datecount=datecount+1
; a=BinaryPeekStr(db,offset,datenamesize)
; hitcount=BinaryPeek4(db,offset+datenamesize)
; FileWrite(rep,strcat(strfix(a,' ',16),hitcount))
;next
BoxText("Building daily section")
datecount=0
if BinaryPeek(db,0)!=0
datecount=1
a=BinaryPeekStr(db,0,datenamesize)
hitcount=BinaryPeek4(db,datenamesize)
FileWrite(rep,strcat(strfix(a,' ',44),hitcount))
endif
offset=datetotalsize
while offset<datealloc
offset=BinaryIndex(db,offset,"",@fwdscan)
if !offset then break
datecount=datecount+1
a=BinaryPeekStr(db,offset,datenamesize)
hitcount=BinaryPeek4(db,offset+datenamesize)
FileWrite(rep,strcat(strfix(a,' ',44),hitcount))
offset=offset+datetotalsize
end while
ttt4=GetTickCount()
FileWrite(rep,'</pre><hr><h2><A NAME="Hourly">Hourly Statistics</A></H2><pre>')
;boxText("Writing summaries by Time")
BoxText("Building hourly section")
FileWrite(rep,'Hour Files Sent')
FileWrite(rep,'------------------------------')
for f=0 to 23
FileWrite(rep,strcat(strfix(f,' ',22),t%f%))
next
ttt5=GetTickCount()
FileWrite(rep,'</pre><hr><h2><A NAME="Dest">Total by Destination</A></H2><pre>')
boxText("Writing Destination summaries")
FileWrite(rep,'Destination Files Sent')
FileWrite(rep,'--------------------------------------------')
compcount=0
if BinaryPeek(cb,0)!=0
compcount=1
a=BinaryPeekStr(cb,0,cns)
hitcount=BinaryPeek4(cb,cns)
FileWrite(rep,strcat(strfix(a,' ',44),hitcount))
endif
offset=cts
while offset<compalloc
offset=BinaryIndex(cb,offset,"",@fwdscan)
if !offset then break
compcount=compcount+1
a=BinaryPeekStr(cb,offset,cns)
hitcount=BinaryPeek4(cb,offset+cns)
FileWrite(rep,strcat(strfix(a,' ',44),hitcount))
offset=offset+cts
end while
ttt6=GetTickCount()
FileWrite(rep,'</pre><hr><h2><A NAME="Archive">Total by file</A></H2><pre>')
FileWrite(rep,'FileName Hit Count')
FileWrite(rep,'--------------------------------------------------------------------------')
BoxText("Writing File Summaries")
;filecount=0
;for offset= 0 to filealloc by fts
; if BinaryPeek(fb,offset)==0 then continue
; filecount=filecount+1
; a=BinaryPeekStr(fb,offset,fns)
; hitcount=BinaryPeek4(fb,offset+fns)
; FileWrite(rep,strcat(strfix(a,' ',74),hitcount))
;next
filecount=0
if BinaryPeek(fb,0)!=0
filecount=1
a=BinaryPeekStr(fb,0,fns)
hitcount=BinaryPeek4(fb,fns)
FileWrite(rep,strcat(strfix(a,' ',74),hitcount))
endif
offset=fts
while offset<filealloc
offset=BinaryIndex(fb,offset,"",@fwdscan)
if !offset then break
filecount=filecount+1
a=BinaryPeekStr(fb,offset,fns)
hitcount=BinaryPeek4(fb,offset+fns)
FileWrite(rep,strcat(strfix(a,' ',74),hitcount))
offset=offset+fts
end while
ttt7=GetTickCount()
BoxText("Building report trailer")
fileWrite(rep,'</pre><p><A HREF="http://webbatch.windowware.com">Statistics by WinBatch</A>')
byebye=GetExactTime()
fileWrite(rep,'<pre>Begin %begin% <br>Finish %byebye%</pre>')
FileWrite(rep,'<pre>')
FileWrite(rep,'Date: Hash Entries Avail %datemaxcount% Used %datecount%')
FileWrite(rep,'Comp: Hash Entries Avail %cmc% Used %compcount%')
FileWrite(rep,'File: Hash Entries Avail %filemaxcount% Used %filecount%')
ttt8=GetTickCount()
FileWrite(rep,'ttt0=0')
for x=1 to 8
y=x-1
ttt=ttt%x%-ttt%y%
FileWrite(rep,'ttt%x%=%ttt%')
next
FileWrite(rep,'</pre>')
fileWrite(rep,'</body></html>')
fileClose(rep)
boxText("Closing out processing")
BinaryFree(fb)
BinaryFree(cb)
BinaryFree(db)
exit
:dafirst
firstd=dadate
firstt=datime
return
:daerror
error%rsltcode%=error%rsltcode%+1
return
Article ID: W12452
Filename: NCSA Log report maker script.txt