iLenceel

def processUrl(self, stringUrl):
        """
        Sets the url that the parser is working on.
        Raises an exception if we can't open it.
        """
        self.robotParser = robotparser.RobotFileParser()
        # check access rights, if not ok raise exception
        if not self.__canVisitSite(stringUrl):
            logging.info
            raise Exception
        # create the HTTP request
        req = self.__createRequest(stringUrl)
        # open the url and set our site to the opened url
        site = urllib2.urlopen(req)
 
        if (not (site.headers.type == 'text/html')) and
           (not (site.headers.type == 'application/x-javascript')):
               logging.info
               raise Exception
        logging.info('successfully opened %s'; % stringUrl)
        self.currentSite = site

def setSite(self, stringUrl, content):
        """
        Sets the current site url and content for the extractor.
        @param stringUrl: The url of the site being analyzed.
        @param content: The html content of the site.
        """
        self.stringUrl = stringUrl #.rstrip('/ ')
        preDomain = urlparse.urlparse(self.stringUrl)
        self.domain = urlparse.urlunparse((preDomain[0],
                      preDomain[1],'', '', '', ''))
 
        fullContent = content.read()
        doc = H.document_fromstring(fullContent.decode('utf-8'))
        doc.make_links_absolute(extractBaseUrl(stringUrl))
        self.parsedContent = doc
        self.rawContent = H.tostring(doc, pretty_print=True,
                         encoding=unicode,method='html')