WinBatch Tech Support Home

Database Search

If you can't find the information using the categories below, post a question over in our WinBatch Tech Support Forum.

TechHome

WebBatch
plus
plus
plus
plus
plus

Can't find the information you are looking for here? Then leave a message over on our WinBatch Tech Support Forum.

;This WinBatch script - with some luck - will process
;standard NCSA Log files.  It will not process MSII files.
;It should not be hard to do, I just have not done it it

;This script takes a LOG file and makes a HTML file summary,
;suitable for posting.

; For BIG log files....Use FAST machines.  This is a real
; number cruncher type script using advanced WinBatch features.
; It has been optimized for speed.  However WinBatch was not really
; designed for this task, so special-purpose log analyzers will
; of course beat it hands down.  But then this works, and it can
; quietly run in the background while you do other work.

logfile=AskFileName("Web Report Writer","","*.log|Log Files|*.*|All Files|","*.log",1)
reportfn=strcat(FilePath(logfile),FileRoot(logfile),".HTML")

;-------------------------------------------------

BoxOpen("Processing %logfile% -> %reportfn%","Initializing")

;pause(logfile,reportfn)
;WEBBATCH NCSA Script reporter
param1=0
param2=0
param3=0
param4=0
param5=0
param6=0
param7=0
param8=0
param9=0
paramA=0
paramB=0
paramC=0
x=@string
x=@ascending

begin=GetExactTime()
Exclusive(@on)
ttt0=GetTickCount()

tstamp=GetExactTime()


cmc=20000
cns=96
compcountersize=4
cts=cns+compcountersize
compalloc=cmc*cts
cb=BinaryAlloc(compalloc+cts)
BinaryEODSet(cb,compalloc+cts)
cno=0

filemaxcount=10000
fns=196
filecountersize=4
fts=fns+filecountersize
filealloc=filemaxcount*fts
fb=BinaryAlloc(filealloc+fts)
BinaryEODSet(fb,filealloc+fts)
fno=0

datemaxcount=400
datenamesize=12
datecountersize=4
datetotalsize=datenamesize+datecountersize
datealloc=datemaxcount*datetotalsize
db=BinaryAlloc(datealloc+datetotalsize)
datenameoffset=0
dash="XXX"
dindex=-1

for x=0 to 31
   day%x%=0
   t%x%=0
next
error404=0
error500=0
tb=0

logsize=FileSize(logfile)+2000
log=BinaryAlloc(logsize)
logsize=BinaryRead(log,logfile)

;Tricky part here.  Take an educated guess at the line delimiter.
; this code should pick out LF, CR or CRLF
; It will get fooled and fail on LFCR or other combos

crcnt=BinaryStrCnt(log,0,min(logsize-1,5000),@CR)
lfcnt=BinaryStrCnt(log,0,min(logsize-1,5000),@LF)

if crcnt==0 && lfcnt!=0 
   delim=@LF
else
   if crcnt!=0 && lfcnt==0 
      delim=@cr
   else
      if abs(crcnt-lfcnt)<3 
         delim=@CRLF
      else
         Message("Ooops","Cannot determine line delimiters on file")
         exit
      endif
   endif
endif   
delimsize=strlen(delim)


logindex=0
logcount=0


lc=0
ttt1=GetTickCount()
while 1
newIndex=BinaryIndex(log,logindex,delim,@fwdscan)
if newindex==0 then break
line=BinaryPeekStr(log,logindex,newindex-logindex)
logindex=newindex+delimsize
ParseData(line)
;---
cname=param1
xx=strreplace(param4,'[','')
dadate=ItemExtract(1,xx,':')
dayom=ItemExtract(1,dadate,"/")+0
y=strlen(dadate)
datime=strsub(xx,y+2,-1)
rsltcode=param7
if param8=="-" then numbytes=0
else numbytes=param8
dafn=strlower(ItemExtract(2,param6,' '))
if strindex("?",dafn,0,@fwdscan) then dafn=strcat(ItemExtract(1,dafn,'?')," ")


;---
if !lc then gosub dafirst
if rsltcode==404  || rsltcode==500 then gosub daerror
lc=lc+1
BoxText("Processing line %lc%")
tb=tb+numbytes
o=BinaryHashRec(cb,cts,cno,cns,cname)
BinaryIncr4(cb,o+cns)
o=BinaryHashRec(fb,fts,fno,fns,dafn)
BinaryIncr4(fb,o+fns)
if dash!=dadate then dindex=dindex+1
then dash=dadate
o=dindex*datetotalsize
if BinaryPeek(db,o)==0 then BinaryPokeStr(db,o,dadate)
BinaryIncr4(db, o+datenamesize)
day%dayom%=day%dayom%+1
tx=strsub(datime,1,2)
tx=tx+0
t%tx%=t%tx%+1
end while


BoxText("Building report headers")


ttt2=GetTickCount()

lastd=dadate
lastt=datime

;BinaryWrite(cb,"comp.before")
;BinaryWrite(fb,"file.before")
BinarySort(cb,cts,cno,cns,@STRING|@ASCENDING)
;BinaryWrite(cb,"comp.after1")
;BinaryWrite(fb,"file.after1")
BinarySort(fb,fts,fno,fns,@STRING|@ASCENDING)
;BinaryWrite(cb,"comp.after2")
;BinaryWrite(fb,"file.after2")
               ;Also @DECENDING  @WORD1  @WORD2  @WORD4  @FLOAT8
ttt3=GetTickCount()



;Display(5,'Processing','Complete')
;boxText("Writing HTML Headers")
rep=FileOpen(reportfn,'WRITE')
FileWrite(rep,'<HTML><HEAD><TITLE>WebBatch Statistics Report</TITLE></HEAD><BODY>')
FileWrite(rep,'<h1>WebBatch Statistics Report</h1>')
FileWrite(rep,strcat('<em> Last Updated: ',TimeDate(),'</em>'))
FileWrite(rep,'<ul>')
FileWrite(rep,'<LI><A HREF="#Daily">Daily Statistics</A>')
FileWrite(rep,'<LI><A HREF="#Hourly">Hourly Statistics</A>')
FileWrite(rep,'<LI><A HREF="#Dest">Total by Destination</A>')
FileWrite(rep,'<LI><A HREF="#Archive">Total by File</A>')
FileWrite(rep,'</ul>')
FileWrite(rep,'<h2>Total for Summary Period</h2><b>%firstd%</b> %firstt%<br><blockquote>to</blockquote><b>%lastd%</b> %lastt%<br><pre>')
FileWrite(rep,'Files Transmitted            %lc%')
FileWrite(rep,'Bytes Transmitted            %tb%')
FileWrite(rep,'File not found errors        %error404%')
FileWrite(rep,'CGI Errors                   %error500%')
FileWrite(rep,'</pre><hr><h2><A NAME="Daily">Daily Statistics</A></H2><pre>')
;boxText("Writing Summaries by Date")
FileWrite(rep,'Date                Files Sent')
FileWrite(rep,'------------------------------')



;datecount=0
;for offset= 0 to datealloc by datetotalsize
;   if BinaryPeek(db,offset)==0 then continue
;   datecount=datecount+1
;   a=BinaryPeekStr(db,offset,datenamesize)
;   hitcount=BinaryPeek4(db,offset+datenamesize)
;   FileWrite(rep,strcat(strfix(a,' ',16),hitcount)) 
;next
BoxText("Building daily section")

datecount=0
if BinaryPeek(db,0)!=0
   datecount=1
   a=BinaryPeekStr(db,0,datenamesize)
   hitcount=BinaryPeek4(db,datenamesize)
   FileWrite(rep,strcat(strfix(a,' ',44),hitcount)) 
endif
offset=datetotalsize
while offset<datealloc
   offset=BinaryIndex(db,offset,"",@fwdscan)
   if !offset then break
   datecount=datecount+1
   a=BinaryPeekStr(db,offset,datenamesize)
   hitcount=BinaryPeek4(db,offset+datenamesize)
   FileWrite(rep,strcat(strfix(a,' ',44),hitcount))
   offset=offset+datetotalsize 
end while

ttt4=GetTickCount()



FileWrite(rep,'</pre><hr><h2><A NAME="Hourly">Hourly Statistics</A></H2><pre>')
;boxText("Writing summaries by Time")
BoxText("Building hourly section")

FileWrite(rep,'Hour                Files Sent')
FileWrite(rep,'------------------------------')

for f=0 to 23
   FileWrite(rep,strcat(strfix(f,' ',22),t%f%))
   next
ttt5=GetTickCount()

FileWrite(rep,'</pre><hr><h2><A NAME="Dest">Total by Destination</A></H2><pre>')
boxText("Writing Destination summaries")

FileWrite(rep,'Destination                       Files Sent')
FileWrite(rep,'--------------------------------------------')


compcount=0
if BinaryPeek(cb,0)!=0
   compcount=1
   a=BinaryPeekStr(cb,0,cns)
   hitcount=BinaryPeek4(cb,cns)
   FileWrite(rep,strcat(strfix(a,' ',44),hitcount)) 
endif
offset=cts
while offset<compalloc
   offset=BinaryIndex(cb,offset,"",@fwdscan)
   if !offset then break
   compcount=compcount+1
   a=BinaryPeekStr(cb,offset,cns)
   hitcount=BinaryPeek4(cb,offset+cns)
   FileWrite(rep,strcat(strfix(a,' ',44),hitcount))
   offset=offset+cts 
end while

ttt6=GetTickCount()


FileWrite(rep,'</pre><hr><h2><A NAME="Archive">Total by file</A></H2><pre>')

FileWrite(rep,'FileName                                                        Hit Count')
FileWrite(rep,'--------------------------------------------------------------------------')
BoxText("Writing File Summaries")

;filecount=0
;for offset= 0 to filealloc by fts
;   if BinaryPeek(fb,offset)==0 then continue
;   filecount=filecount+1
;   a=BinaryPeekStr(fb,offset,fns)
;   hitcount=BinaryPeek4(fb,offset+fns)
;   FileWrite(rep,strcat(strfix(a,' ',74),hitcount)) 
;next
filecount=0
if BinaryPeek(fb,0)!=0
   filecount=1
   a=BinaryPeekStr(fb,0,fns)
   hitcount=BinaryPeek4(fb,fns)
   FileWrite(rep,strcat(strfix(a,' ',74),hitcount)) 
endif
offset=fts
while offset<filealloc
   offset=BinaryIndex(fb,offset,"",@fwdscan)
   if !offset then break
   filecount=filecount+1
   a=BinaryPeekStr(fb,offset,fns)
   hitcount=BinaryPeek4(fb,offset+fns)
   FileWrite(rep,strcat(strfix(a,' ',74),hitcount))
   offset=offset+fts 
end while

ttt7=GetTickCount()
BoxText("Building report trailer")


fileWrite(rep,'</pre><p><A HREF="http://webbatch.windowware.com">Statistics by WinBatch</A>')
byebye=GetExactTime()
fileWrite(rep,'<pre>Begin %begin% <br>Finish %byebye%</pre>')
FileWrite(rep,'<pre>')
FileWrite(rep,'Date: Hash Entries Avail %datemaxcount%  Used %datecount%')
FileWrite(rep,'Comp: Hash Entries Avail %cmc%  Used %compcount%')
FileWrite(rep,'File: Hash Entries Avail %filemaxcount%  Used %filecount%')
ttt8=GetTickCount()

FileWrite(rep,'ttt0=0')
for x=1 to 8
y=x-1
ttt=ttt%x%-ttt%y%
FileWrite(rep,'ttt%x%=%ttt%')
next

FileWrite(rep,'</pre>')
fileWrite(rep,'</body></html>')
fileClose(rep)
boxText("Closing out processing")



BinaryFree(fb)
BinaryFree(cb)
BinaryFree(db)


exit

:dafirst
     firstd=dadate
     firstt=datime
return

:daerror
     error%rsltcode%=error%rsltcode%+1
return



Article ID:   W12452
Filename:   NCSA Log report maker script.txt
File Created: 2010:02:17:11:58:08
Last Updated: 2001:03:06:11:51:30