1 from sys import argv
  2 from os import makedirs,unlink,sep,mkdir
  3 from os.path import dirname,exists,isdir,splitext
  4 from string import replace,find,lower
  5 from htmllib import HTMLParser
  6 from urllib import urlretrieve
  7 from urlparse import urlparse,urljoin
  8 from formatter import DumbWriter,AbstractFormatter
  9 from cStringIO import StringIO
 10 
 11 
 12 class Retriever(object):
 13     def __init__(self,url):
 14         self.url = url
 15         self.file = \'E:\install\Python27\\\' + self.filename(url)
 16 
 17     def filename(self,url,deffile=\'index.htm\'):
 18         parsedurl = urlparse(url,\'http:\',0)
 19         path = parsedurl[1] + parsedurl[2]
 20         ext = splitext(path) # seperate ext name
 21         if ext[1] == \'\':
 22             if path[-1] == \'/\':
 23                 path += deffile
 24             else:
 25                 path += \'/\' + deffile
 26 
 27         ldir = dirname(path)
 28         if sep != \'/\':
 29             ldir = replace(ldir,\'/\',sep)
 30         if not isdir(ldir):
 31             if exists(ldir): unlink(ldir)
 32             makedirs(ldir)
 33         return path
 34 
 35     def download(self):
 36         try:
 37             retval = urlretrieve(self.url,self.file)
 38         except IOError:
 39             retval = (\'*** ERROR: invalid URL "%s"\' %\
 40                 self.url)
 41         return retval
 42 
 43     def parseAndGetLinks(self):
 44         self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
 45         self.parser.feed(open(self.file).read())
 46         self.parser.close()
 47         return self.parser.anchorlist
 48 
 49 class Crawler(object):
 50     count = 0 # static downloaded page counter
 51 
 52     def __init__(self,url):
 53         self.q = [url]
 54         self.seen = []
 55         self.dom = urlparse(url)[1]
 56 
 57     def getPage(self,url):
 58         r = Retriever(url)
 59         retval = r.download()
 60         if retval[0] == \'*\':
 61             print retval,\'...skipping parse\'
 62             return
 63         Crawler.count += 1
 64         print \'\n(\',Crawler.count,\')\'
 65         print \'URL:\',url
 66         print \'FILE:\',retval[0]
 67         self.seen.append(url)
 68 
 69 
 70 
 71         links = r.parseAndGetLinks()
 72         for eachLink in links:
 73             if eachLink[:4] != \'http\' and find(eachLink,\'://\') == -1:
 74                 eachLink = urljoin(url,eachLink)
 75 
 76             if find(lower(eachLink),\'mailto:\') != -1:
 77                 print \'...discarded,mailto link\'
 78                 continue
 79             if eachLink not in self.seen:
 80                 if find(eachLink,self.dom) == -1:
 81                     print \'...discarded,not in domain\'
 82                 else:
 83                     if eachLink not in self.q:
 84                         self.q.append(eachLink)
 85                         print \'...new,added to Q\'
 86                     else:
 87                         print \'...discarded,already in Q\'
 88             else:
 89                 print \'...discarded,already processed\'
 90 
 91 
 92 
 93     def go(self):#process links in queue
 94         while self.q:
 95             url = self.q.pop()
 96             self.getPage(url)
 97 
 98 
 99 
100 def main():
101     if len(argv) > 1:
102         url = argv[1]
103 
104     else:
105         try:
106             url = raw_input(\'Enter starting URL:\')
107         except(KeyboardInerrupt,EOFError):
108             url = \'\'
109         if not url: return
110         robot = Crawler(url)
111         robot.go()
112 
113 if __name__ == \'__main__\':
114     main()