<commit_before>import xmlrpclib
import time
import solr

class LabcasClient(object):
    '''
    Python client used to interact with a remote Labcas back-end.
    Mostly for demo and example purposes...
    Available methods are defined in Java class org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManager.
    '''

    def __init__(self,
                 workflowManagerUrl='http://localhost:9001/',
                 fileManagerUrl='http://localhost:9000/',
                 solrUrl='http://localhost:8983/solr/oodt-fm',
                 verbose=False):

        self.workflowManagerServerProxy = xmlrpclib.ServerProxy(workflowManagerUrl, verbose=verbose)
        self.fileManaferServerProxy = xmlrpclib.ServerProxy(fileManagerUrl, verbose=verbose)
        self.solrServerProxy = solr.SolrConnection(solrUrl)

    def getWorkflowsByEvent(self, eventName):
        '''Retrieve a specific workflow by the triggering event.'''

        workflows =  self.workflowManagerServerProxy.workflowmgr.getWorkflowsByEvent(eventName)
        for workflow in workflows:
            self.printWorkflow(workflow)

    def getWorkflowById(self, workflowId):
        '''Retrieve a specific workflow by its unique identifier.'''

        workflow =  self.workflowManagerServerProxy.workflowmgr.getWorkflowById(workflowId)
        self.printWorkflow(workflow)

    def executeWorkflow(self, tasks, metadata):
        '''Submits a dynamic workflow composed of the specified tasks, using the specified metadata.'''

        # FIXME: pass metadata through: s.encode('ascii',errors='ignore')
        return self.workflowManagerServerProxy.workflowmgr.executeDynamicWorkflow(tasks, metadata)

    def waitForCompletion(self, wInstId):
        ''' Monitors a workflow instance until it completes.'''

        # wait for the server to instantiate this workflow before querying it
        time.sleep(4)

        # now use the workflow instance id to check for status, wait until completed
        running_status  = ['CREATED', 'QUEUED', 'STARTED', 'PAUSED']
        pge_task_status = ['STAGING INPUT', 'BUILDING CONFIG FILE', 'PGE EXEC', 'CRAWLING']
        finished_status = ['FINISHED', 'ERROR', 'METMISS']
        while (True):
            response = self.workflowManagerServerProxy.workflowmgr.getWorkflowInstanceById(wInstId)
            status = response['status']
            if status in running_status or status in pge_task_status:
                print 'Workflow instance=%s running with status=%s' % (wInstId, status)
                time.sleep(1)
            elif status in finished_status:
                print 'Workflow instance=%s ended with status=%s' % (wInstId, status)
                break
            else:
                print 'UNRECOGNIZED WORKFLOW STATUS: %s' % status
                break
        print response

    def uploadCollection(self, datasetId, metadata, newVersion=False, inPlace=False):

        # add 'DatasetId' key, value to other metadata
        metadata['DatasetId'] = datasetId

        # optionally request a new version
        if newVersion:
            metadata['NewVersion'] = 'true'

        # NOTE: currently, if you start a named workflow, the XMLRPC interface only returns True/False, not a workflow instance identifier...
        #tf = serverProxy.workflowmgr.handleEvent('labcas-upload', { 'DatasetId':'mydata' } )

        # ... consequently, you must submit an equivalent dynamic workflow, which does return the workflow instance id
        if inPlace:
            wInstId = self.workflowManagerServerProxy.workflowmgr.executeDynamicWorkflow( ['urn:edrn:LabcasUploadInitTask','urn:edrn:LabcasUpload2ExecuteTask'],
                                                                                metadata )
        else:
            wInstId = self.workflowManagerServerProxy.workflowmgr.executeDynamicWorkflow( ['urn:edrn:LabcasUploadInitTask','urn:edrn:LabcasUploadExecuteTask'],
                                                                                metadata )

        # monitor workflow instance
        self.waitForCompletion(wInstId)

    def getProductTypeByName(self, datasetName):

        # retrieve a specific product type by name
        productTypeDict =  self.fileManaferServerProxy.filemgr.getProductTypeByName(datasetName)

        self.printProductType(productTypeDict)

    def getProductTypeById(self, datasetId):

        # retrieve a specific product type by name
        productTypeDict =  self.fileManaferServerProxy.filemgr.getProductTypeById(datasetId)

        self.printProductType(productTypeDict)

    def listProductTypes(self):

        # list all supported product types
        productTypes =  self.fileManaferServerProxy.filemgr.getProductTypes()
        for productTypeDict in productTypes:
            self.printProductType(productTypeDict)

    def listTopLevelProductTypes(self):

        # roots of product type hierarchy (NOT be displayed directly)
        BUILTIN_PRODUCTS = ['GenericFile', 'LabcasProduct','EcasProduct']

        # dictionary of top level product types:
        # key = DatasetId, value = dictionary of properties
        topLevelProductTypes = {}

        # list all supported product types
        productTypes =  self.fileManaferServerProxy.filemgr.getProductTypes()
        for productTypeDict in productTypes:

            # assemble information to be displayed by UI
            name = productTypeDict['name']

            # do NOT include these types in display list
            if name not in BUILTIN_PRODUCTS:

                typeMetadata = productTypeDict['typeMetadata']
                datasetId = typeMetadata['DatasetId'][0]
                try:
                    datasetName = typeMetadata['DatasetName'][0]
                except KeyError:
                    datasetName = None

                description = productTypeDict['description']
                if typeMetadata.get('OrganSite', None):
                    organSite = typeMetadata['OrganSite'][0]
                else:
                    organSite = None
                if typeMetadata.get('LeadPI', None):
                    leadPI = typeMetadata['LeadPI'][0]
                else:
                    leadPI = None

                topLevelProductTypes[datasetId] = { 'name': name,
                                                    'description':  description,
                                                    'datasetId': datasetId,
                                                    'datasetName': datasetName,
                                                    'organSite': organSite,
                                                    'leadPI':leadPI }

        return topLevelProductTypes

    def listProductTypesByParent(self, parentDatasetId):

        childrenProductTypes = []

        # list all supported product types
        productTypes =  self.fileManaferServerProxy.filemgr.getProductTypes()
        for productTypeDict in productTypes:
            try:
                if parentDatasetId in productTypeDict['typeMetadata']['ParentDatasetId']:
                    datasetId =  productTypeDict['typeMetadata']['DatasetId'][0]
                    childrenProductTypes.append(datasetId)
            except KeyError:
                pass

        return childrenProductTypes

    def printProductType(self, productTypeDict):
        print 'PRODUCT TYPE: %s' % productTypeDict['name']
        for key, value in productTypeDict.items():
            # dictionary: typeMetadata = {'DataCustodianEmail': ['dsidrans@jhmi.edu'], 'DataDisclaimer': [...], ..}
            if key=='typeMetadata':
                print '\t%s =' % key
                for _key, _value in value.items():
                    print '\t\t%s = %s' % (_key, _value)
            else:
                print '\t%s = %s' % (key, value)

    def listProducts(self, productType):

        # query for all products of this type (i.e. all files of this dataset), all versions
        #response = self.solrServerProxy.query('*:*', fq=['DatasetId:%s' % datasetId], start=0)
        #print "\nNumber of files found: %s (all versions)" % response.numFound
        #for result in response.results:
        #    self.printProduct(result)

        # query for all possible versions of this dataset
        response = self.solrServerProxy.query('*:*', fq=['CAS.ProductTypeName:%s' % productType], start=0, rows=0, facet='true', facet_field='DatasetVersion')
        versions = response.facet_counts['facet_fields']['DatasetVersion']
        last_version = 0
        for key, value in versions.items():
            # NOTE: facet keys span the whole index, but their counts are specific to this search
            if int(value)>0:
                print "\nDataset Version number %s has %s files" % (key, value)
                if int(key) > last_version:
                    last_version = int(key)

        # query for all files for a specific version
        response = self.solrServerProxy.query('*:*', fq=['CAS.ProductTypeName:%s' % productType,'DatasetVersion:%s' % last_version ], start=0)
        print "\nLatest version: %s, number of files: %s, listing them all:" % (last_version, response.numFound)
        for result in response.results:
            self.printProduct(result)

    def printProduct(self, result):
        '''Utility function to print out the product metadata'''

        print '\nProduct ID=%s' % result['id']
        for key, values in result.items():
            print '\tProduct metadata key=%s values=%s' % (key, values)

    def printWorkflow(self, workflowDict):
        '''Utiliyu function to print out a workflow.'''

        print workflowDict
        print "Workflow id=%s name=%s" % (workflowDict['id'], workflowDict['name'])
        for task in workflowDict['tasks']:
            print "Task: %s" % task
<commit_msg>Making labcas_client more tolerant for XMLRPC errors.<commit_after>import xmlrpclib
import time
import solr

class LabcasClient(object):
    '''
    Python client used to interact with a remote Labcas back-end.
    Mostly for demo and example purposes...
    Available methods are defined in Java class org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManager.
    '''

    def __init__(self,
                 workflowManagerUrl='http://localhost:9001/',
                 fileManagerUrl='http://localhost:9000/',
                 solrUrl='http://localhost:8983/solr/oodt-fm',
                 verbose=False):

        self.workflowManagerServerProxy = xmlrpclib.ServerProxy(workflowManagerUrl, verbose=verbose)
        self.fileManaferServerProxy = xmlrpclib.ServerProxy(fileManagerUrl, verbose=verbose)
        self.solrServerProxy = solr.SolrConnection(solrUrl)

    def getWorkflowsByEvent(self, eventName):
        '''Retrieve a specific workflow by the triggering event.'''

        workflows =  self.workflowManagerServerProxy.workflowmgr.getWorkflowsByEvent(eventName)
        for workflow in workflows:
            self.printWorkflow(workflow)

    def getWorkflowById(self, workflowId):
        '''Retrieve a specific workflow by its unique identifier.'''

        workflow =  self.workflowManagerServerProxy.workflowmgr.getWorkflowById(workflowId)
        self.printWorkflow(workflow)

    def executeWorkflow(self, tasks, metadata):
        '''Submits a dynamic workflow composed of the specified tasks, using the specified metadata.'''

        # FIXME: pass metadata through: s.encode('ascii',errors='ignore')
        return self.workflowManagerServerProxy.workflowmgr.executeDynamicWorkflow(tasks, metadata)

    def waitForCompletion(self, wInstId):
        ''' Monitors a workflow instance until it completes.'''

        # wait for the server to instantiate this workflow before querying it
        time.sleep(4)

        # now use the workflow instance id to check for status, wait until completed
        running_status  = ['CREATED', 'QUEUED', 'STARTED', 'PAUSED']
        pge_task_status = ['STAGING INPUT', 'BUILDING CONFIG FILE', 'PGE EXEC', 'CRAWLING']
        finished_status = ['FINISHED', 'ERROR', 'METMISS']
        while (True):
            try:
                response = self.workflowManagerServerProxy.workflowmgr.getWorkflowInstanceById(wInstId)
                status = response['status']
                if status in running_status or status in pge_task_status:
                    print 'Workflow instance=%s running with status=%s' % (wInstId, status)
                    time.sleep(1)
                elif status in finished_status:
                    print 'Workflow instance=%s ended with status=%s' % (wInstId, status)
                    break
                else:
                    print 'UNRECOGNIZED WORKFLOW STATUS: %s' % status
                    break
            except xmlrpclib.Fault as e:
                # must ignore XML-RPC exeptions that often arise when querying OODT for a specific workflow
                # just try again with the same workflow identifier
                print e
        print response

    def uploadCollection(self, datasetId, metadata, newVersion=False, inPlace=False):

        # add 'DatasetId' key, value to other metadata
        metadata['DatasetId'] = datasetId

        # optionally request a new version
        if newVersion:
            metadata['NewVersion'] = 'true'

        # NOTE: currently, if you start a named workflow, the XMLRPC interface only returns True/False, not a workflow instance identifier...
        #tf = serverProxy.workflowmgr.handleEvent('labcas-upload', { 'DatasetId':'mydata' } )

        # ... consequently, you must submit an equivalent dynamic workflow, which does return the workflow instance id
        if inPlace:
            wInstId = self.workflowManagerServerProxy.workflowmgr.executeDynamicWorkflow( ['urn:edrn:LabcasUploadInitTask','urn:edrn:LabcasUpload2ExecuteTask'],
                                                                                metadata )
        else:
            wInstId = self.workflowManagerServerProxy.workflowmgr.executeDynamicWorkflow( ['urn:edrn:LabcasUploadInitTask','urn:edrn:LabcasUploadExecuteTask'],
                                                                                metadata )

        # monitor workflow instance
        self.waitForCompletion(wInstId)

    def getProductTypeByName(self, datasetName):

        # retrieve a specific product type by name
        productTypeDict =  self.fileManaferServerProxy.filemgr.getProductTypeByName(datasetName)

        self.printProductType(productTypeDict)

    def getProductTypeById(self, datasetId):

        # retrieve a specific product type by name
        productTypeDict =  self.fileManaferServerProxy.filemgr.getProductTypeById(datasetId)

        self.printProductType(productTypeDict)

    def listProductTypes(self):

        # list all supported product types
        productTypes =  self.fileManaferServerProxy.filemgr.getProductTypes()
        for productTypeDict in productTypes:
            self.printProductType(productTypeDict)

    def listTopLevelProductTypes(self):

        # roots of product type hierarchy (NOT be displayed directly)
        BUILTIN_PRODUCTS = ['GenericFile', 'LabcasProduct','EcasProduct']

        # dictionary of top level product types:
        # key = DatasetId, value = dictionary of properties
        topLevelProductTypes = {}

        # list all supported product types
        productTypes =  self.fileManaferServerProxy.filemgr.getProductTypes()
        for productTypeDict in productTypes:

            # assemble information to be displayed by UI
            name = productTypeDict['name']

            # do NOT include these types in display list
            if name not in BUILTIN_PRODUCTS:

                typeMetadata = productTypeDict['typeMetadata']
                datasetId = typeMetadata['DatasetId'][0]
                try:
                    datasetName = typeMetadata['DatasetName'][0]
                except KeyError:
                    datasetName = None

                description = productTypeDict['description']
                if typeMetadata.get('OrganSite', None):
                    organSite = typeMetadata['OrganSite'][0]
                else:
                    organSite = None
                if typeMetadata.get('LeadPI', None):
                    leadPI = typeMetadata['LeadPI'][0]
                else:
                    leadPI = None

                topLevelProductTypes[datasetId] = { 'name': name,
                                                    'description':  description,
                                                    'datasetId': datasetId,
                                                    'datasetName': datasetName,
                                                    'organSite': organSite,
                                                    'leadPI':leadPI }

        return topLevelProductTypes

    def listProductTypesByParent(self, parentDatasetId):

        childrenProductTypes = []

        # list all supported product types
        productTypes =  self.fileManaferServerProxy.filemgr.getProductTypes()
        for productTypeDict in productTypes:
            try:
                if parentDatasetId in productTypeDict['typeMetadata']['ParentDatasetId']:
                    datasetId =  productTypeDict['typeMetadata']['DatasetId'][0]
                    childrenProductTypes.append(datasetId)
            except KeyError:
                pass

        return childrenProductTypes

    def printProductType(self, productTypeDict):
        print 'PRODUCT TYPE: %s' % productTypeDict['name']
        for key, value in productTypeDict.items():
            # dictionary: typeMetadata = {'DataCustodianEmail': ['dsidrans@jhmi.edu'], 'DataDisclaimer': [...], ..}
            if key=='typeMetadata':
                print '\t%s =' % key
                for _key, _value in value.items():
                    print '\t\t%s = %s' % (_key, _value)
            else:
                print '\t%s = %s' % (key, value)

    def listProducts(self, productType):

        # query for all products of this type (i.e. all files of this dataset), all versions
        #response = self.solrServerProxy.query('*:*', fq=['DatasetId:%s' % datasetId], start=0)
        #print "\nNumber of files found: %s (all versions)" % response.numFound
        #for result in response.results:
        #    self.printProduct(result)

        # query for all possible versions of this dataset
        response = self.solrServerProxy.query('*:*', fq=['CAS.ProductTypeName:%s' % productType], start=0, rows=0, facet='true', facet_field='DatasetVersion')
        versions = response.facet_counts['facet_fields']['DatasetVersion']
        last_version = 0
        for key, value in versions.items():
            # NOTE: facet keys span the whole index, but their counts are specific to this search
            if int(value)>0:
                print "\nDataset Version number %s has %s files" % (key, value)
                if int(key) > last_version:
                    last_version = int(key)

        # query for all files for a specific version
        response = self.solrServerProxy.query('*:*', fq=['CAS.ProductTypeName:%s' % productType,'DatasetVersion:%s' % last_version ], start=0)
        print "\nLatest version: %s, number of files: %s, listing them all:" % (last_version, response.numFound)
        for result in response.results:
            self.printProduct(result)

    def printProduct(self, result):
        '''Utility function to print out the product metadata'''

        print '\nProduct ID=%s' % result['id']
        for key, values in result.items():
            print '\tProduct metadata key=%s values=%s' % (key, values)

    def printWorkflow(self, workflowDict):
        '''Utiliyu function to print out a workflow.'''

        print workflowDict
        print "Workflow id=%s name=%s" % (workflowDict['id'], workflowDict['name'])
        for task in workflowDict['tasks']:
            print "Task: %s" % task
<|endoftext|>


<commit_before>from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.views.decorators.cache import cache_control
from django.views.generic.detail import DetailView
import json

from bulbs.content.views import BaseContentDetailView
from bulbs.poll.models import Poll, Answer
from bulbs.poll.serializers import PollPublicSerializer

class PollDetailView(BaseContentDetailView):
    model = Poll

class MergedPollDataView(DetailView):
    model = Poll

    def render_to_response(self, context, **response_kwargs):
        serializer = PollPublicSerializer(self.object)
        response = HttpResponse(json.dumps(serializer.data),
            content_type="application/json"
        )
        response["Access-Control-Allow-Origin"] = "*"
        return response

poll_detail = cache_control(max_age=600)(PollDetailView.as_view())
get_merged_poll_data = cache_control(max_age=600)(MergedPollDataView.as_view())
<commit_msg>testing ordering filter with rest framework<commit_after>from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.views.decorators.cache import cache_control
from django.views.generic.detail import DetailView
from rest_framework import filters
from rest_framework import generic
import json

from bulbs.content.views import BaseContentDetailView
from bulbs.poll.models import Poll, Answer
from bulbs.poll.serializers import PollPublicSerializer

class PollDetailView(BaseContentDetailView):
    model = Poll
    filter_backends = (filters.OrderingFilter,)
    ordering_fields= "__all__"

class MergedPollDataView(DetailView):
    model = Poll

    def render_to_response(self, context, **response_kwargs):
        serializer = PollPublicSerializer(self.object)
        response = HttpResponse(json.dumps(serializer.data),
            content_type="application/json"
        )
        response["Access-Control-Allow-Origin"] = "*"
        return response

poll_detail = cache_control(max_age=600)(PollDetailView.as_view())
get_merged_poll_data = cache_control(max_age=600)(MergedPollDataView.as_view())
<|endoftext|>


<commit_before>import scrapy
import logging
import datetime

from DailyStar.items import DailyStarItem
from newspaper import Article

from DailyStar.Helpers.CustomNERTagger import Tagger

#indico API_KEY 8ee6432e7dc137740c40c0af8d7e9d05

# Change these paths according to your system
STANFORD_NER_PATH = 'C:\StanfordParser\stanford-ner-2015-12-09\stanford-ner.jar'
STANFORD_CLASSIFIER_PATH = 'C:\StanfordParser\stanford-ner-2015-12-09\classifiers\english.all.3class.distsim.crf.ser.gz'

# Using elasticsearch
from elasticsearch import Elasticsearch

es = Elasticsearch()

class DailyStarSpider(scrapy.Spider):
    name = 'dailystar'

    def increase_day_by_one(self, d):
        d += datetime.timedelta(days=1)
        return d

    # def __init__(self, start_day, start_month, start_year):
    #     self.start_day = int(start_day)
    #     self.start_month = int(start_month)
    #     self.start_year = int(start_year)

    def start_requests(self):
        self.start_year = 2016
        self.start_month = 1
        self.start_day = 1
        self.start_date = datetime.date(self.start_year , self.start_month , self.start_day)
        self.url = 'http://www.thedailystar.net/newspaper?' + self.start_date.__str__()

        # Creating the Tagger object
        self.tagger = Tagger(classifier_path=STANFORD_CLASSIFIER_PATH, ner_path=STANFORD_NER_PATH)

        self.id = 0

        yield scrapy.Request(self.url, self.parse)

    def parse(self, response):
        self.main_url = 'http://www.thedailystar.net'
        self.baseurl = 'http://www.thedailystar.net/newspaper?date='

        self.main_selection = response.xpath("//h5")

        for sel in self.main_selection:
            news_item = DailyStarItem()
            news_item['newspaper_section'] = sel.xpath("../../../../../../../div[1]/h2/text()").extract_first()
            news_item['url'] = self.main_url + sel.xpath("a/@href").extract_first()
            news_item['title'] = sel.xpath("a/text()").extract_first().strip()

            request = scrapy.Request(news_item['url'], callback=self.parseNews)

            request.meta['news_item'] = news_item
            yield request

        self.start_date = self.increase_day_by_one(self.start_date)

        self.next_page = self.baseurl + self.start_date.__str__()

        try:
            self.logger.info("TRYING")
            yield scrapy.Request(self.next_page, callback=self.parse)
        except:
            self.logger.info("PROBLEM")
            self.start_date = self.increase_day_by_one(self.start_date)
            self.next_page = self.baseurl + self.start_date.__str__()
            yield scrapy.Request(self.next_page, callback=self.parse)

    def parseNews(self, response):

        self.id += 1

        news_item = response.meta['news_item']

        #Getting the Article
        paragraphs = response.xpath("//div[@class='field-body view-mode-teaser']//p/text()").extract()
        news_item['article'] = ''.join([para.strip() for para in paragraphs])

        # Getting bottom tag line
        news_item['bottom_tag_line'] = response.xpath("//h2[@class='h5 margin-bottom-zero']/em/text()").extract_first()
        # Getting top tag line
        news_item['top_tag_line'] = response.xpath("//h4[@class='uppercase']/text()").extract_first()

        # Getting the published time
        news_item = self.getPublishedTime(news_item, response)

        # Getting the image source and captions
        news_item['images'] = response.xpath("//div[@class='caption']/../img/@src").extract()
        news_item['image_captions'] = response.xpath("//div[@class='caption']/text()").extract()

        # Get the breadcrumb
        news_item['breadcrumb'] = response.xpath("//div[@class='breadcrumb']//span[@itemprop='name']/text()").extract()

        # Get reporter
        news_item['reporter'] = response.xpath("//div[@class='author-name margin-bottom-big']/span/a/text()").extract_first()

        # Get the summary and keywords using 'newspaper' package
        # [WARNING : This section slows down the overall scraping process]
        article = Article(url=news_item['url'])
        article.download()
        article.parse()
        article.nlp()

        news_item['generated_summary'] = article.summary
        news_item['generated_keywords'] = article.keywords

        # Tagging the article
        try:
            self.tagger.entity_group(news_item['article'])
        except:
            print "NER Tagger exception"
        # Getting the ner tags
        news_item['ner_person'] = self.tagger.PERSON
        news_item['ner_organization'] = self.tagger.ORGANIZATION
        news_item['ner_time'] = self.tagger.TIME
        news_item['ner_percent'] = self.tagger.PERCENT
        news_item['ner_money'] = self.tagger.MONEY
        news_item['ner_location'] = self.tagger.LOCATION

        # ML tags
        news_item['ml_tags'] = None

        news_item['sentiment'] = self.tagger.get_indico_sentiment(news_item['article'])

        doc = {
            "news_url" : news_item['url'],
            "reporter" : news_item['reporter'],
            "published" : news_item['published_date'],
            "title" : news_item['title'],
            "content" : news_item['article'],
            "top_tagline" : news_item['top_tag_line'],
            "bottom_tagline" : news_item['bottom_tag_line'],
            "images" : news_item['images'],
            "image_captions" : news_item['image_captions'],
            "breadcrumb" : news_item['breadcrumb'],
            "sentiment" : news_item['sentiment'],
            "ml_tags" : None,
            "section" : news_item['newspaper_section'],
            "ner_person" : news_item['ner_person'],
            "ner_organization" : news_item['ner_organization'],
            "ner_money" : news_item['ner_money'],
            "ner_time" : news_item['ner_time'],
            "ner_location" : news_item['ner_location'],
            "ner_percent" : news_item['ner_percent'],
            "generated_keywords" : news_item['generated_keywords'],
            "generated_summary" : news_item['generated_summary'],
            "timestamp" : datetime.datetime.now(),
        }

        res = es.index(index="dailystar-index", doc_type='news', id=self.id, body=doc)

        # Data can be collected as csv also
        yield doc

    def getPublishedTime(self, news_item, response):
        dt = response.xpath("//meta[@itemprop='datePublished']/@content").extract_first()
        converted_dt = datetime.datetime.strptime( dt.split("+")[0], "%Y-%m-%dT%H:%M:%S")
        formatted_dt = converted_dt.strftime("%Y-%m-%d %I:%M %p")
        news_item['published_date'] = formatted_dt
        return news_item

<commit_msg>newspaper index fix<commit_after>import scrapy
import logging
import datetime

from DailyStar.items import DailyStarItem
from newspaper import Article

from DailyStar.Helpers.CustomNERTagger import Tagger

#indico API_KEY 8ee6432e7dc137740c40c0af8d7e9d05

# Change these paths according to your system
STANFORD_NER_PATH = 'C:\StanfordParser\stanford-ner-2015-12-09\stanford-ner.jar'
STANFORD_CLASSIFIER_PATH = 'C:\StanfordParser\stanford-ner-2015-12-09\classifiers\english.all.3class.distsim.crf.ser.gz'

# Using elasticsearch
from elasticsearch import Elasticsearch

es = Elasticsearch()

class DailyStarSpider(scrapy.Spider):
    name = 'dailystar'

    def increase_day_by_one(self, d):
        d += datetime.timedelta(days=1)
        return d

    # def __init__(self, start_day, start_month, start_year):
    #     self.start_day = int(start_day)
    #     self.start_month = int(start_month)
    #     self.start_year = int(start_year)

    def start_requests(self):
        self.start_year = 2016
        self.start_month = 1
        self.start_day = 1
        self.start_date = datetime.date(self.start_year , self.start_month , self.start_day)
        self.url = 'http://www.thedailystar.net/newspaper?' + self.start_date.__str__()

        # Creating the Tagger object
        self.tagger = Tagger(classifier_path=STANFORD_CLASSIFIER_PATH, ner_path=STANFORD_NER_PATH)

        self.id = 0

        yield scrapy.Request(self.url, self.parse)

    def parse(self, response):
        self.main_url = 'http://www.thedailystar.net'
        self.baseurl = 'http://www.thedailystar.net/newspaper?date='

        self.main_selection = response.xpath("//h5")

        for sel in self.main_selection:
            news_item = DailyStarItem()
            news_item['newspaper_section'] = sel.xpath("../../../../../../../div[1]/h2/text()").extract_first()
            news_item['url'] = self.main_url + sel.xpath("a/@href").extract_first()
            news_item['title'] = sel.xpath("a/text()").extract_first().strip()

            request = scrapy.Request(news_item['url'], callback=self.parseNews)

            request.meta['news_item'] = news_item
            yield request

        self.start_date = self.increase_day_by_one(self.start_date)

        self.next_page = self.baseurl + self.start_date.__str__()

        try:
            self.logger.info("TRYING")
            yield scrapy.Request(self.next_page, callback=self.parse)
        except:
            self.logger.info("PROBLEM")
            self.start_date = self.increase_day_by_one(self.start_date)
            self.next_page = self.baseurl + self.start_date.__str__()
            yield scrapy.Request(self.next_page, callback=self.parse)

    def parseNews(self, response):

        self.id += 1

        news_item = response.meta['news_item']

        #Getting the Article
        paragraphs = response.xpath("//div[@class='field-body view-mode-teaser']//p/text()").extract()
        news_item['article'] = ''.join([para.strip() for para in paragraphs])

        # Getting bottom tag line
        news_item['bottom_tag_line'] = response.xpath("//h2[@class='h5 margin-bottom-zero']/em/text()").extract_first()
        # Getting top tag line
        news_item['top_tag_line'] = response.xpath("//h4[@class='uppercase']/text()").extract_first()

        # Getting the published time
        news_item = self.getPublishedTime(news_item, response)

        # Getting the image source and captions
        news_item['images'] = response.xpath("//div[@class='caption']/../img/@src").extract()
        news_item['image_captions'] = response.xpath("//div[@class='caption']/text()").extract()

        # Get the breadcrumb
        news_item['breadcrumb'] = response.xpath("//div[@class='breadcrumb']//span[@itemprop='name']/text()").extract()

        # Get reporter
        news_item['reporter'] = response.xpath("//div[@class='author-name margin-bottom-big']/span/a/text()").extract_first()

        # Get the summary and keywords using 'newspaper' package
        # [WARNING : This section slows down the overall scraping process]
        article = Article(url=news_item['url'])
        article.download()
        article.parse()
        article.nlp()

        news_item['generated_summary'] = article.summary
        news_item['generated_keywords'] = article.keywords

        # Tagging the article
        try:
            self.tagger.entity_group(news_item['article'])
        except:
            print "NER Tagger exception"
        # Getting the ner tags
        news_item['ner_person'] = self.tagger.PERSON
        news_item['ner_organization'] = self.tagger.ORGANIZATION
        news_item['ner_time'] = self.tagger.TIME
        news_item['ner_percent'] = self.tagger.PERCENT
        news_item['ner_money'] = self.tagger.MONEY
        news_item['ner_location'] = self.tagger.LOCATION

        # ML tags
        news_item['ml_tags'] = None

        news_item['sentiment'] = self.tagger.get_indico_sentiment(news_item['article'])

        doc = {
            "news_url" : news_item['url'],
            "reporter" : news_item['reporter'],
            "published" : news_item['published_date'],
            "title" : news_item['title'],
            "content" : news_item['article'],
            "top_tagline" : news_item['top_tag_line'],
            "bottom_tagline" : news_item['bottom_tag_line'],
            "images" : news_item['images'],
            "image_captions" : news_item['image_captions'],
            "breadcrumb" : news_item['breadcrumb'],
            "sentiment" : news_item['sentiment'],
            "ml_tags" : None,
            "section" : news_item['newspaper_section'],
            "ner_person" : news_item['ner_person'],
            "ner_organization" : news_item['ner_organization'],
            "ner_money" : news_item['ner_money'],
            "ner_time" : news_item['ner_time'],
            "ner_location" : news_item['ner_location'],
            "ner_percent" : news_item['ner_percent'],
            "generated_keywords" : news_item['generated_keywords'],
            "generated_summary" : news_item['generated_summary'],
            "timestamp" : datetime.datetime.now(),
        }

        res = es.index(index="newspaper_index", doc_type='news', id=self.id, body=doc)

        # Data can be collected as csv also
        yield doc

    def getPublishedTime(self, news_item, response):
        dt = response.xpath("//meta[@itemprop='datePublished']/@content").extract_first()
        converted_dt = datetime.datetime.strptime( dt.split("+")[0], "%Y-%m-%dT%H:%M:%S")
        formatted_dt = converted_dt.strftime("%Y-%m-%d %I:%M %p")
        news_item['published_date'] = formatted_dt
        return news_item

<|endoftext|>


<commit_before># Lint as: python3
# Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base stub component launcher for launching component executors and stub executors in process."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
from typing import Any, Dict, List, Text, Type

from tfx import types
from tfx.components.base import base_executor
from tfx.experimental.pipeline_testing import base_stub_executor
from tfx.orchestration.launcher import in_process_component_launcher

class BaseStubComponentLauncher(
    in_process_component_launcher.InProcessComponentLauncher):
  """Responsible for launching a stub executor.

  A stub component launcher class inheriting BaseStubComponentLauncher can be
  supported by the pipeline. This class should be defined in a separate python
  module because the class is pickled by module path in beam and imported by
  module path in KFP.
  The executor will be launched in the same process of the rest of the
  component, i.e. its driver and publisher.
  """

  stubbed_component_map = ...  # type: Dict[Text, Type[base_stub_executor.BaseStubExecutor]]
  test_data_dir = ...  # type: Text

  @classmethod
  def initialize(
      cls,
      test_data_dir: Text,
      stubbed_component_ids: List[Text],
      stubbed_component_map: Dict[Text, Type[base_stub_executor.BaseStubExecutor]]  # pylint: disable=line-too-long
  ):
    """Intializes variables in the stub launcher class.

    For beam pipeline, stub launcher class inheriting this class is defined in
    tfx.experimental.pipeline_testing.stub_component_launcher.py

    For KFP, stub launcher class inheriting this class is defined in
    tfx.experimental.templates.taxi.launcher.stub_component_launcher.py.

    These classes then can be used to launch stub executors in the pipeline.

    For example,

    class MyPusherStubExecutor(base_stub_executor.BaseStubExecutor){...}
    class MyTransformStubExecutor(base_stub_executor.BaseStubExecutor){...}

    stub_component_launcher.StubComponentLauncher.initialize(
                test_data_dir,
                stubbed_component_ids = ['CsvExampleGen'],
                stubbed_component_map = {
                    'Transform': MyTransformStubExecutor,
                    'Pusher': MyPusherStubExecutor})
    PipelineConfig(
        supported_launcher_classes=[
            stub_component_launcher.StubComponentLauncher
        ],
    )

    The method initializes the necessary class variables for the
    StubComponentLauncher class, including stubbed_component_ids and
    stubbed_component_map holding custom executor classes, which
    users may define differently per component.

    Args:
      test_data_dir: The directory where pipeline outputs are recorded
        (pipeline_recorder.py).
      stubbed_component_ids: List of component ids that should be replaced
        with aBaseStubExecutor.
      stubbed_component_map: Dictionary holding user-defined stub executor.
        These user-defined stub executors must inherit from
        base_stub_executor.BaseStubExecutor.

    Returns:
      None
    """
    cls.stubbed_component_map = dict(stubbed_component_map)
    for component_id in stubbed_component_ids:
      cls.stubbed_component_map[component_id] = base_stub_executor.BaseStubExecutor  # pylint: disable=line-too-long
    cls.test_data_dir = test_data_dir

  def _run_executor(self, execution_id: int,
                    input_dict: Dict[Text, List[types.Artifact]],
                    output_dict: Dict[Text, List[types.Artifact]],
                    exec_properties: Dict[Text, Any]) -> None:
    """Execute underlying component implementation."""
    component_id = self._component_info.component_id
    if component_id not in self.stubbed_component_map:
      super(BaseStubComponentLauncher,
            self)._run_executor(execution_id, input_dict, output_dict,
                                exec_properties)
    else:
      executor_context = base_executor.BaseExecutor.Context(
          beam_pipeline_args=self._beam_pipeline_args,
          tmp_dir=os.path.join(self._pipeline_info.pipeline_root, '.temp', ''),
          unique_id=str(execution_id))
      executor = self.stubbed_component_map[component_id](component_id,
                                                          self.test_data_dir,
                                                          executor_context)
      executor.Do(input_dict, output_dict, exec_properties)
<commit_msg>nit fix<commit_after># Lint as: python3
# Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base stub component launcher for launching component executors and stub executors in process."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
from typing import Any, Dict, List, Text, Type

from tfx import types
from tfx.components.base import base_executor
from tfx.experimental.pipeline_testing import base_stub_executor
from tfx.orchestration.launcher import in_process_component_launcher

class BaseStubComponentLauncher(
    in_process_component_launcher.InProcessComponentLauncher):
  """Responsible for launching a stub executor.

  A stub component launcher class inheriting BaseStubComponentLauncher can be
  supported by the pipeline. This class should be defined in a separate python
  module because the class is pickled by module path in beam and imported by
  module path in KFP.
  The executor will be launched in the same process of the rest of the
  component, i.e. its driver and publisher.
  """

  stubbed_component_map = ...  # type: Dict[Text, Type[base_stub_executor.BaseStubExecutor]]
  test_data_dir = ...  # type: Text

  @classmethod
  def initialize(
      cls,
      test_data_dir: Text,
      stubbed_component_ids: List[Text],
      stubbed_component_map: Dict[Text, Type[base_stub_executor.BaseStubExecutor]]  # pylint: disable=line-too-long
  ):
    """Intializes variables in the stub launcher class.

    For beam pipeline, stub launcher class inheriting this class is defined in
    tfx.experimental.pipeline_testing.stub_component_launcher.py

    For KFP, stub launcher class inheriting this class is defined in
    tfx.experimental.templates.taxi.launcher.stub_component_launcher.py.

    These classes then can be used to launch stub executors in the pipeline.

    For example,

    class MyPusherStubExecutor(base_stub_executor.BaseStubExecutor){...}
    class MyTransformStubExecutor(base_stub_executor.BaseStubExecutor){...}

    stub_component_launcher.StubComponentLauncher.initialize(
                test_data_dir,
                stubbed_component_ids = ['CsvExampleGen'],
                stubbed_component_map = {
                    'Transform': MyTransformStubExecutor,
                    'Pusher': MyPusherStubExecutor})
    PipelineConfig(
        supported_launcher_classes=[
            stub_component_launcher.StubComponentLauncher
        ],
    )

    The method initializes the necessary class variables for the
    BaseStubComponentLauncher class, including stubbed_component_ids and
    stubbed_component_map holding custom executor classes, which
    users may define differently per component.

    Args:
      test_data_dir: The directory where pipeline outputs are recorded
        (pipeline_recorder.py).
      stubbed_component_ids: List of component ids that should be replaced
        with aBaseStubExecutor.
      stubbed_component_map: Dictionary holding user-defined stub executor.
        These user-defined stub executors must inherit from
        base_stub_executor.BaseStubExecutor.

    Returns:
      None
    """
    cls.stubbed_component_map = dict(stubbed_component_map)
    for component_id in stubbed_component_ids:
      cls.stubbed_component_map[component_id] = base_stub_executor.BaseStubExecutor  # pylint: disable=line-too-long
    cls.test_data_dir = test_data_dir

  def _run_executor(self, execution_id: int,
                    input_dict: Dict[Text, List[types.Artifact]],
                    output_dict: Dict[Text, List[types.Artifact]],
                    exec_properties: Dict[Text, Any]) -> None:
    """Execute underlying component implementation."""
    component_id = self._component_info.component_id
    if component_id not in self.stubbed_component_map:
      super(BaseStubComponentLauncher,
            self)._run_executor(execution_id, input_dict, output_dict,
                                exec_properties)
    else:
      executor_context = base_executor.BaseExecutor.Context(
          beam_pipeline_args=self._beam_pipeline_args,
          tmp_dir=os.path.join(self._pipeline_info.pipeline_root, '.temp', ''),
          unique_id=str(execution_id))
      executor = self.stubbed_component_map[component_id](component_id,
                                                          self.test_data_dir,
                                                          executor_context)
      executor.Do(input_dict, output_dict, exec_properties)
<|endoftext|>


<commit_before># pylint: disable=R0201

"""Output classes"""

import sys
import json
import uuid
import logging

import bumblebee.store
import bumblebee.util

MAX_PERCENTS = 100.
CHARS = 8
HBARS = [
    u"\u2581",
    u"\u2582",
    u"\u2583",
    u"\u2584",
    u"\u2585",
    u"\u2586",
    u"\u2587",
    u"\u2588"]
VBARS = [
    u"\u258f",
    u"\u258e",
    u"\u258d",
    u"\u258c",
    u"\u258b",
    u"\u258a",
    u"\u2589",
    u"\u2588"]
BRAILLE = {
    (0, 0): u" ",
    (1, 0): u"\u2840",
    (2, 0): u"\u2844",
    (3, 0): u"\u2846",
    (4, 0): u"\u2847",
    (0, 1): u"\u2880",
    (0, 2): u"\u28a0",
    (0, 3): u"\u28b0",
    (0, 4): u"\u28b8",
    (1, 1): u"\u28c0",
    (2, 1): u"\u28c4",
    (3, 1): u"\u28c6",
    (4, 1): u"\u28c7",
    (1, 2): u"\u28e0",
    (2, 2): u"\u28e4",
    (3, 2): u"\u28e6",
    (4, 2): u"\u28e7",
    (1, 3): u"\u28f0",
    (2, 3): u"\u28f4",
    (3, 3): u"\u28f6",
    (4, 3): u"\u28f7",
    (1, 4): u"\u28f8",
    (2, 4): u"\u28fc",
    (3, 4): u"\u28fe",
    (4, 4): u"\u28ff"
}

log = logging.getLogger(__name__)

def scrollable(func):
    def wrapper(module, widget):
        text = func(module, widget)
        if not text:
            return text
        width = widget.get("theme.width", int(module.parameter("width", 30)))
        if bumblebee.util.asbool(module.parameter("scrolling.makewide", "true")):
            widget.set("theme.minwidth", "A"*width)
        if width < 0:
            return text
        if len(text) <= width:
            return text
        # we need to shorten

        try:
            bounce = int(module.parameter("scrolling.bounce", 1))
        except ValueError:
            bounce = 1
        try:
            scroll_speed = int(module.parameter("scrolling.speed", 1))
        except ValueError:
            scroll_speed = 1
        start = widget.get("scrolling.start", -1)
        direction = widget.get("scrolling.direction", "right")
        start += scroll_speed if direction == "right" else -(scroll_speed)

        if width + start > len(text) + (scroll_speed -1):
            if bounce:
                widget.set("scrolling.direction", "left")
            else:
                start = 0
        elif start <= 0:
            if bounce:
                widget.set("scrolling.direction", "right")
            else:
                start = len(text)
        widget.set("scrolling.start", start)
        text = text[start:width+start]

        return text
    return wrapper

class Bar(object):
    """superclass"""
    bars = None

    def __init__(self, value):
        """
            Args:

                value (float): value between 0. and 100. meaning percents
        """
        self.value = value

class HBar(Bar):
    """horizontal bar (1 char)"""
    bars = HBARS

    def __init__(self, value):
        """
            Args:

                value (float): value between 0. and 100. meaning percents
        """
        super(HBar, self).__init__(value)
        self.step = MAX_PERCENTS / CHARS

    def get_char(self):
        """
            Decide which char to draw

            Return: str
        """
        for i in range(CHARS):
            left = i * self.step
            right = (i + 1) * self.step
            if left <= self.value < right:
                return self.bars[i]
        return self.bars[-1]

def hbar(value):
    """wrapper function"""
    return HBar(value).get_char()

class VBar(Bar):
    """vertical bar (can be more than 1 char)"""
    bars = VBARS

    def __init__(self, value, width=1):
        """
            Args:

                value (float): value between 0. and 100. meaning percents

                width (int): width
        """
        super(VBar, self).__init__(value)
        self.step = MAX_PERCENTS / (CHARS * width)
        self.width = width

    def get_chars(self):
        """
            Decide which char to draw

            Return: str
        """
        if self.value == 100:
            return self.bars[-1] * self.width
        if self.width == 1:
            for i in range(CHARS):
                left = i * self.step
                right = (i + 1) * self.step
                if left <= self.value < right:
                    return self.bars[i]
        else:
            full_parts = int(self.value // (self.step * CHARS))
            remainder = self.value - full_parts * self.step * CHARS
            empty_parts = self.width - full_parts
            if remainder >= 0:
                empty_parts -= 1
            part_vbar = VBar(remainder * self.width)  # scale to width
            chars = self.bars[-1] * full_parts
            chars += part_vbar.get_chars()
            chars += " " * empty_parts
            return chars

def vbar(value, width):
    """wrapper function"""
    return VBar(value, width).get_chars()

class BrailleGraph(object):
    """
        graph using Braille chars
        scaled to passed values
    """
    def __init__(self, values):
        """
            Args:

                values (list): list of values
        """
        self.values = values
        # length of values list must be even
        # because one Braille char displays two values
        if len(self.values) % 2 == 1:
            self.values.append(0)
        self.steps = self.get_steps()
        self.parts = [tuple(self.steps[i:i+2])
                      for i in range(len(self.steps))[::2]]

    @staticmethod
    def get_height(value, unit):
        """
            Compute height of a value relative to unit

            Args:

                value (number): value

                unit (number): unit
        """
        if value < unit / 10.:
            return 0
        elif value <= unit:
            return 1
        elif value <= unit * 2:
            return 2
        elif value <= unit * 3:
            return 3
        else:
            return 4

    def get_steps(self):
        """
            Convert the list of values to a list of steps

            Return: list
        """
        maxval = max(self.values)
        unit = maxval / 4.
        if unit == 0:
            return [0] * len(self.values)
        stepslist = []
        for value in self.values:
            stepslist.append(self.get_height(value, unit))
        return stepslist

    def get_chars(self):
        """
            Decide which chars to draw

            Return: str
        """
        chars = []
        for part in self.parts:
            chars.append(BRAILLE[part])
        return "".join(chars)

def bgraph(values):
    """wrapper function"""
    return BrailleGraph(values).get_chars()

class Widget(bumblebee.store.Store):
    """Represents a single visible block in the status bar"""
    def __init__(self, full_text="", name=""):
        super(Widget, self).__init__()
        self._full_text = full_text
        self.module = None
        self._module = None
        self._minimized = False
        self.name = name
        self.id = str(uuid.uuid4())

    def get_module(self):
        return self._module

    def toggle_minimize(self):
        self._minimized = not self._minimized

    def link_module(self, module):
        """Set the module that spawned this widget

        This is done outside the constructor to avoid having to
        pass in the module name in every concrete module implementation"""
        self.module = module.name
        self._module = module

    def cls(self):
        if not self._module:
            return None
        return self._module.__module__.replace("bumblebee.modules.", "")

    def state(self):
        """Return the widget's state"""
        if self._module and hasattr(self._module, "state"):
            states = self._module.state(self)
            if not isinstance(states, list):
                return [states]
            return states
        return []

    def full_text(self, value=None):
        """Set or retrieve the full text to display in the widget"""
        if value:
            self._full_text = value
        else:
            if self._minimized:
                return u"\u2026"
            if callable(self._full_text):
                return self._full_text(self)
            else:
                return self._full_text

class I3BarOutput(object):
    """Manage output according to the i3bar protocol"""
    def __init__(self, theme, config=None):
        self._theme = theme
        self._widgets = []
        self._started = False
        self._config = config

    def started(self):
        return self._started

    def start(self):
        """Print start preamble for i3bar protocol"""
        self._started = True
        sys.stdout.write(json.dumps({"version": 1, "click_events": True}) + "\n[\n")

    def stop(self):
        """Finish i3bar protocol"""
        sys.stdout.write("]\n")

    def draw(self, widget, module=None, engine=None):
        """Draw a single widget"""
        full_text = widget.full_text()
        if widget.get_module() and widget.get_module().hidden():
            return
        if widget.get_module() and widget.get_module().name in self._config.autohide():
            if not any(state in widget.state() for state in ["warning", "critical"]):
                return
        padding = self._theme.padding(widget)

        prefix = self._theme.prefix(widget, padding)
        suffix = self._theme.suffix(widget, padding)

        if self._config.markup() == "pango":
            # add prefix/suffix colors
            fg = self._theme.prefix_fg(widget)
            bg = self._theme.prefix_bg(widget)
            prefix = "<span {} {}>{}</span>".format(
                "foreground='{}'".format(fg) if fg else "",
                "background='{}'".format(bg) if bg else "",
                prefix
            )

        if prefix:
            full_text = u"{}{}".format(prefix, full_text)
        if suffix:
            full_text = u"{}{}".format(full_text, suffix)

        separator = self._theme.separator(widget)
        if separator:
            self._widgets.append({
                u"full_text": separator,
                "separator": False,
                "color": self._theme.separator_fg(widget),
                "background": self._theme.separator_bg(widget),
                "separator_block_width": self._theme.separator_block_width(widget),
            })
        width = self._theme.minwidth(widget)

        if width:
            full_text = full_text.ljust(len(width) + len(prefix) + len(suffix))

        markup = "none" if not self._config else self._config.markup()

        if markup == "pango":
            full_text = full_text.replace("&", "&")

        self._widgets.append({
            u"full_text": full_text,
            "color": self._theme.fg(widget),
            "background": self._theme.bg(widget),
            "separator_block_width": self._theme.separator_block_width(widget),
            "separator": True if separator is None else False,
            "min_width": None,
#            "min_width": width + "A"*(len(prefix) + len(suffix)) if width else None,
            "align": self._theme.align(widget),
            "instance": widget.id,
            "name": module.id,
            "markup": markup,
        })

    def begin(self):
        """Start one output iteration"""
        self._widgets = []
        self._theme.reset()

    def flush(self):
        """Flushes output"""
        widgets = self._widgets
        if self._config and self._config.reverse():
            widgets = list(reversed(widgets))
        sys.stdout.write(json.dumps(widgets))
        if len(self._config.unused_keys()) > 0:
            for key in self._config.unused_keys():
                log.warning("unused parameter {} - please check the documentation of the affected module to ensure the parameter exists".format(key))

    def end(self):
        """Finalizes output"""
        sys.stdout.write(",\n")
        sys.stdout.flush()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>[doc] improve docstring<commit_after># pylint: disable=R0201

"""Output classes"""

import sys
import json
import uuid
import logging

import bumblebee.store
import bumblebee.util

MAX_PERCENTS = 100.
CHARS = 8
HBARS = [
    u"\u2581",
    u"\u2582",
    u"\u2583",
    u"\u2584",
    u"\u2585",
    u"\u2586",
    u"\u2587",
    u"\u2588"]
VBARS = [
    u"\u258f",
    u"\u258e",
    u"\u258d",
    u"\u258c",
    u"\u258b",
    u"\u258a",
    u"\u2589",
    u"\u2588"]
BRAILLE = {
    (0, 0): u" ",
    (1, 0): u"\u2840",
    (2, 0): u"\u2844",
    (3, 0): u"\u2846",
    (4, 0): u"\u2847",
    (0, 1): u"\u2880",
    (0, 2): u"\u28a0",
    (0, 3): u"\u28b0",
    (0, 4): u"\u28b8",
    (1, 1): u"\u28c0",
    (2, 1): u"\u28c4",
    (3, 1): u"\u28c6",
    (4, 1): u"\u28c7",
    (1, 2): u"\u28e0",
    (2, 2): u"\u28e4",
    (3, 2): u"\u28e6",
    (4, 2): u"\u28e7",
    (1, 3): u"\u28f0",
    (2, 3): u"\u28f4",
    (3, 3): u"\u28f6",
    (4, 3): u"\u28f7",
    (1, 4): u"\u28f8",
    (2, 4): u"\u28fc",
    (3, 4): u"\u28fe",
    (4, 4): u"\u28ff"
}

log = logging.getLogger(__name__)

def scrollable(func):
    def wrapper(module, widget):
        text = func(module, widget)
        if not text:
            return text
        width = widget.get("theme.width", int(module.parameter("width", 30)))
        if bumblebee.util.asbool(module.parameter("scrolling.makewide", "true")):
            widget.set("theme.minwidth", "A"*width)
        if width < 0:
            return text
        if len(text) <= width:
            return text
        # we need to shorten

        try:
            bounce = int(module.parameter("scrolling.bounce", 1))
        except ValueError:
            bounce = 1
        try:
            scroll_speed = int(module.parameter("scrolling.speed", 1))
        except ValueError:
            scroll_speed = 1
        start = widget.get("scrolling.start", -1)
        direction = widget.get("scrolling.direction", "right")
        start += scroll_speed if direction == "right" else -(scroll_speed)

        if width + start > len(text) + (scroll_speed -1):
            if bounce:
                widget.set("scrolling.direction", "left")
            else:
                start = 0
        elif start <= 0:
            if bounce:
                widget.set("scrolling.direction", "right")
            else:
                start = len(text)
        widget.set("scrolling.start", start)
        text = text[start:width+start]

        return text
    return wrapper

class Bar(object):
    """superclass"""
    bars = None

    def __init__(self, value):
        """
            Args:

                value (float): value between 0. and 100. meaning percents
        """
        self.value = value

class HBar(Bar):
    """horizontal bar (1 char)"""
    bars = HBARS

    def __init__(self, value):
        """
            Args:

                value (float): value between 0. and 100. meaning percents
        """
        super(HBar, self).__init__(value)
        self.step = MAX_PERCENTS / CHARS

    def get_char(self):
        """
            Decide which char to draw

            Return: str
        """
        for i in range(CHARS):
            left = i * self.step
            right = (i + 1) * self.step
            if left <= self.value < right:
                return self.bars[i]
        return self.bars[-1]

def hbar(value):
    """wrapper function"""
    return HBar(value).get_char()

class VBar(Bar):
    """vertical bar (can be more than 1 char)"""
    bars = VBARS

    def __init__(self, value, width=1):
        """
            Args:

                value (float): value between 0. and 100. meaning percents

                width (int): width
        """
        super(VBar, self).__init__(value)
        self.step = MAX_PERCENTS / (CHARS * width)
        self.width = width

    def get_chars(self):
        """
            Decide which char to draw

            Return: str
        """
        if self.value == 100:
            return self.bars[-1] * self.width
        if self.width == 1:
            for i in range(CHARS):
                left = i * self.step
                right = (i + 1) * self.step
                if left <= self.value < right:
                    return self.bars[i]
        else:
            full_parts = int(self.value // (self.step * CHARS))
            remainder = self.value - full_parts * self.step * CHARS
            empty_parts = self.width - full_parts
            if remainder >= 0:
                empty_parts -= 1
            part_vbar = VBar(remainder * self.width)  # scale to width
            chars = self.bars[-1] * full_parts
            chars += part_vbar.get_chars()
            chars += " " * empty_parts
            return chars

def vbar(value, width):
    """wrapper function"""
    return VBar(value, width).get_chars()

class BrailleGraph(object):
    """
        graph using Braille chars
        scaled to passed values
    """
    def __init__(self, values):
        """
            Args:

                values (list): list of values
        """
        self.values = values
        # length of values list must be even
        # because one Braille char displays two values
        if len(self.values) % 2 == 1:
            self.values.append(0)
        self.steps = self.get_steps()
        self.parts = [tuple(self.steps[i:i+2])
                      for i in range(len(self.steps))[::2]]

    @staticmethod
    def get_height(value, unit):
        """
            Compute height of a value relative to unit

            Args:

                value (number): value

                unit (number): unit
        """
        if value < unit / 10.:
            return 0
        elif value <= unit:
            return 1
        elif value <= unit * 2:
            return 2
        elif value <= unit * 3:
            return 3
        else:
            return 4

    def get_steps(self):
        """
            Convert the list of values to a list of steps

            Return: list
        """
        maxval = max(self.values)
        unit = maxval / 4.
        if unit == 0:
            return [0] * len(self.values)
        stepslist = []
        for value in self.values:
            stepslist.append(self.get_height(value, unit))
        return stepslist

    def get_chars(self):
        """
            Decide which chars to draw

            Return: str
        """
        chars = []
        for part in self.parts:
            chars.append(BRAILLE[part])
        return "".join(chars)

def bgraph(values):
    """wrapper function"""
    return BrailleGraph(values).get_chars()

class Widget(bumblebee.store.Store):
    """Represents a single visible block in the status bar"""
    def __init__(self, full_text="", name=""):
        super(Widget, self).__init__()
        self._full_text = full_text
        self.module = None
        self._module = None
        self._minimized = False
        self.name = name
        self.id = str(uuid.uuid4())

    def get_module(self):
        return self._module

    def toggle_minimize(self):
        self._minimized = not self._minimized

    def link_module(self, module):
        """Set the module that spawned this widget

        This is done outside the constructor to avoid having to
        pass in the module name in every concrete module implementation"""
        self.module = module.name
        self._module = module

    def cls(self):
        if not self._module:
            return None
        return self._module.__module__.replace("bumblebee.modules.", "")

    def state(self):
        """Return the widget's state"""
        if self._module and hasattr(self._module, "state"):
            states = self._module.state(self)
            if not isinstance(states, list):
                return [states]
            return states
        return []

    def full_text(self, value=None):
        """Set or retrieve the full text to display in the widget"""
        if value:
            self._full_text = value
        else:
            if self._minimized:
                return u"\u2026"
            if callable(self._full_text):
                return self._full_text(self)
            else:
                return self._full_text

class I3BarOutput(object):
    """Manage output according to the i3bar protocol"""
    def __init__(self, theme, config=None):
        self._theme = theme
        self._widgets = []
        self._started = False
        self._config = config

    def started(self):
        return self._started

    def start(self):
        """Print start preamble for i3bar protocol"""
        self._started = True
        sys.stdout.write(json.dumps({"version": 1, "click_events": True}) + "\n[\n")

    def stop(self):
        """Finish i3bar protocol"""
        sys.stdout.write("]\n")

    def draw(self, widget, module=None, engine=None):
        """
            Draw a single widget

            Note: technically, this method doesn't draw anything. It only adds
            blocks of JSON text to self._widgets: one for separator, if the
            theme contains a separator and one for the widget itself
        """
        full_text = widget.full_text()
        if widget.get_module() and widget.get_module().hidden():
            return
        if widget.get_module() and widget.get_module().name in self._config.autohide():
            if not any(state in widget.state() for state in ["warning", "critical"]):
                return
        padding = self._theme.padding(widget)

        prefix = self._theme.prefix(widget, padding)
        suffix = self._theme.suffix(widget, padding)

        if self._config.markup() == "pango":
            # add prefix/suffix colors
            fg = self._theme.prefix_fg(widget)
            bg = self._theme.prefix_bg(widget)
            prefix = "<span {} {}>{}</span>".format(
                "foreground='{}'".format(fg) if fg else "",
                "background='{}'".format(bg) if bg else "",
                prefix
            )

        if prefix:
            full_text = u"{}{}".format(prefix, full_text)
        if suffix:
            full_text = u"{}{}".format(full_text, suffix)

        separator = self._theme.separator(widget)
        if separator:
            self._widgets.append({
                u"full_text": separator,
                "separator": False,
                "color": self._theme.separator_fg(widget),
                "background": self._theme.separator_bg(widget),
                "separator_block_width": self._theme.separator_block_width(widget),
            })
        width = self._theme.minwidth(widget)

        if width:
            full_text = full_text.ljust(len(width) + len(prefix) + len(suffix))

        markup = "none" if not self._config else self._config.markup()

        if markup == "pango":
            full_text = full_text.replace("&", "&")

        self._widgets.append({
            u"full_text": full_text,
            "color": self._theme.fg(widget),
            "background": self._theme.bg(widget),
            "separator_block_width": self._theme.separator_block_width(widget),
            "separator": True if separator is None else False,
            "min_width": None,
#            "min_width": width + "A"*(len(prefix) + len(suffix)) if width else None,
            "align": self._theme.align(widget),
            "instance": widget.id,
            "name": module.id,
            "markup": markup,
        })

    def begin(self):
        """Start one output iteration"""
        self._widgets = []
        self._theme.reset()

    def flush(self):
        """Flushes output"""
        widgets = self._widgets
        if self._config and self._config.reverse():
            widgets = list(reversed(widgets))
        sys.stdout.write(json.dumps(widgets))
        if len(self._config.unused_keys()) > 0:
            for key in self._config.unused_keys():
                log.warning("unused parameter {} - please check the documentation of the affected module to ensure the parameter exists".format(key))

    def end(self):
        """Finalizes output"""
        sys.stdout.write(",\n")
        sys.stdout.flush()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before># Copyright 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import logging
import os
import shutil
import tempfile
import unittest

from telemetry.core import util
from telemetry import decorators
from telemetry.internal.browser import browser as browser_module
from telemetry.internal.browser import browser_finder
from telemetry.internal.platform import gpu_device
from telemetry.internal.platform import gpu_info
from telemetry.internal.platform import system_info
from telemetry.internal.util import path
from telemetry.testing import browser_test_case
from telemetry.testing import options_for_unittests
from telemetry.timeline import tracing_config

import mock

class IntentionalException(Exception):
  pass

class BrowserTest(browser_test_case.BrowserTestCase):
  def testBrowserCreation(self):
    self.assertEquals(1, len(self._browser.tabs))

    # Different browsers boot up to different things.
    assert self._browser.tabs[0].url

  @decorators.Enabled('has tabs')
  def testNewCloseTab(self):
    existing_tab = self._browser.tabs[0]
    self.assertEquals(1, len(self._browser.tabs))
    existing_tab_url = existing_tab.url

    new_tab = self._browser.tabs.New()
    self.assertEquals(2, len(self._browser.tabs))
    self.assertEquals(existing_tab.url, existing_tab_url)
    self.assertEquals(new_tab.url, 'about:blank')

    new_tab.Close()
    self.assertEquals(1, len(self._browser.tabs))
    self.assertEquals(existing_tab.url, existing_tab_url)

  def testMultipleTabCalls(self):
    self._browser.tabs[0].Navigate(self.UrlOfUnittestFile('blank.html'))
    self._browser.tabs[0].WaitForDocumentReadyStateToBeInteractiveOrBetter()

  def testTabCallByReference(self):
    tab = self._browser.tabs[0]
    tab.Navigate(self.UrlOfUnittestFile('blank.html'))
    self._browser.tabs[0].WaitForDocumentReadyStateToBeInteractiveOrBetter()

  @decorators.Enabled('has tabs')
  @decorators.Disabled('win')  # crbug.com/321527
  def testCloseReferencedTab(self):
    self._browser.tabs.New()
    tab = self._browser.tabs[0]
    tab.Navigate(self.UrlOfUnittestFile('blank.html'))
    tab.Close()
    self.assertEquals(1, len(self._browser.tabs))

  @decorators.Enabled('has tabs')
  def testForegroundTab(self):
    # Should be only one tab at this stage, so that must be the foreground tab
    original_tab = self._browser.tabs[0]
    self.assertEqual(self._browser.foreground_tab, original_tab)
    new_tab = self._browser.tabs.New()
    # New tab shouls be foreground tab
    self.assertEqual(self._browser.foreground_tab, new_tab)
    # Make sure that activating the background tab makes it the foreground tab
    original_tab.Activate()
    self.assertEqual(self._browser.foreground_tab, original_tab)
    # Closing the current foreground tab should switch the foreground tab to the
    # other tab
    original_tab.Close()
    self.assertEqual(self._browser.foreground_tab, new_tab)

  def testGetSystemInfo(self):
    if not self._browser.supports_system_info:
      logging.warning(
          'Browser does not support getting system info, skipping test.')
      return

    info = self._browser.GetSystemInfo()

    self.assertTrue(isinstance(info, system_info.SystemInfo))
    self.assertTrue(hasattr(info, 'model_name'))
    self.assertTrue(hasattr(info, 'gpu'))
    self.assertTrue(isinstance(info.gpu, gpu_info.GPUInfo))
    self.assertTrue(hasattr(info.gpu, 'devices'))
    self.assertTrue(len(info.gpu.devices) > 0)
    for g in info.gpu.devices:
      self.assertTrue(isinstance(g, gpu_device.GPUDevice))

  def testGetSystemInfoNotCachedObject(self):
    if not self._browser.supports_system_info:
      logging.warning(
          'Browser does not support getting system info, skipping test.')
      return

    info_a = self._browser.GetSystemInfo()
    info_b = self._browser.GetSystemInfo()
    self.assertFalse(info_a is info_b)

  def testGetSystemTotalMemory(self):
    self.assertTrue(self._browser.memory_stats['SystemTotalPhysicalMemory'] > 0)

  @decorators.Disabled('cros-chrome-guest', 'system-guest')  # chromeos guest
  def testIsTracingRunning(self):
    tracing_controller = self._browser.platform.tracing_controller
    if not tracing_controller.IsChromeTracingSupported():
      return
    self.assertFalse(tracing_controller.is_tracing_running)
    config = tracing_config.TracingConfig()
    config.enable_chrome_trace = True
    tracing_controller.StartTracing(config)
    self.assertTrue(tracing_controller.is_tracing_running)
    tracing_controller.StopTracing()
    self.assertFalse(tracing_controller.is_tracing_running)

class CommandLineBrowserTest(browser_test_case.BrowserTestCase):
  @classmethod
  def CustomizeBrowserOptions(cls, options):
    options.AppendExtraBrowserArgs('--user-agent=telemetry')

  def testCommandLineOverriding(self):
    # This test starts the browser with --user-agent=telemetry. This tests
    # whether the user agent is then set.
    t = self._browser.tabs[0]
    t.Navigate(self.UrlOfUnittestFile('blank.html'))
    t.WaitForDocumentReadyStateToBeInteractiveOrBetter()
    self.assertEquals(t.EvaluateJavaScript('navigator.userAgent'),
                      'telemetry')

class DirtyProfileBrowserTest(browser_test_case.BrowserTestCase):
  @classmethod
  def CustomizeBrowserOptions(cls, options):
    options.profile_type = 'small_profile'

  @decorators.Disabled('chromeos')  # crbug.com/243912
  def testDirtyProfileCreation(self):
    self.assertEquals(1, len(self._browser.tabs))

class BrowserLoggingTest(browser_test_case.BrowserTestCase):
  @classmethod
  def CustomizeBrowserOptions(cls, options):
    options.logging_verbosity = options.VERBOSE_LOGGING

  @decorators.Disabled('chromeos', 'android')
  def testLogFileExist(self):
    self.assertTrue(
       os.path.isfile(self._browser._browser_backend.log_file_path))

def _GenerateBrowserProfile(number_of_tabs):
  """ Generate a browser profile which browser had |number_of_tabs| number of
  tabs opened before it was closed.
      Returns:
        profile_dir: the directory of profile.
  """
  profile_dir = tempfile.mkdtemp()
  options = options_for_unittests.GetCopy()
  options.browser_options.output_profile_path = profile_dir
  browser_to_create = browser_finder.FindBrowser(options)
  with browser_to_create.Create(options) as browser:
    browser.platform.SetHTTPServerDirectories(path.GetUnittestDataDir())
    blank_file_path = os.path.join(path.GetUnittestDataDir(), 'blank.html')
    blank_url = browser.platform.http_server.UrlOf(blank_file_path)
    browser.foreground_tab.Navigate(blank_url)
    browser.foreground_tab.WaitForDocumentReadyStateToBeComplete()
    for _ in xrange(number_of_tabs - 1):
      tab = browser.tabs.New()
      tab.Navigate(blank_url)
      tab.WaitForDocumentReadyStateToBeComplete()
  return profile_dir

class BrowserCreationTest(unittest.TestCase):
  def setUp(self):
    self.mock_browser_backend = mock.MagicMock()
    self.mock_platform_backend = mock.MagicMock()

  def testCleanedUpCalledWhenExceptionRaisedInBrowserCreation(self):
    self.mock_platform_backend.platform.FlushDnsCache.side_effect = (
        IntentionalException('Boom!'))
    with self.assertRaises(IntentionalException):
      browser_module.Browser(
         self.mock_browser_backend, self.mock_platform_backend,
         credentials_path=None)
    self.assertTrue(self.mock_platform_backend.WillCloseBrowser.called)

  def testOriginalExceptionNotSwallow(self):
    self.mock_platform_backend.platform.FlushDnsCache.side_effect = (
        IntentionalException('Boom!'))
    self.mock_platform_backend.WillCloseBrowser.side_effect = (
        IntentionalException('Cannot close browser!'))
    with self.assertRaises(IntentionalException) as context:
      browser_module.Browser(
         self.mock_browser_backend, self.mock_platform_backend,
         credentials_path=None)
    self.assertIn('Boom!', context.exception.message)

class BrowserRestoreSessionTest(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    cls._number_of_tabs = 4
    cls._profile_dir = _GenerateBrowserProfile(cls._number_of_tabs)
    cls._options = options_for_unittests.GetCopy()
    cls._options.browser_options.AppendExtraBrowserArgs(
        ['--restore-last-session'])
    cls._options.browser_options.profile_dir = cls._profile_dir
    cls._browser_to_create = browser_finder.FindBrowser(cls._options)

  @decorators.Enabled('has tabs')
  @decorators.Disabled('chromeos', 'win', 'mac')
  # TODO(nednguyen): Enable this test on windowsn platform
  def testRestoreBrowserWithMultipleTabs(self):
    with self._browser_to_create.Create(self._options) as browser:
      # The number of tabs will be self._number_of_tabs + 1 as it includes the
      # old tabs and a new blank tab.
      expected_number_of_tabs = self._number_of_tabs + 1
      try:
        util.WaitFor(lambda: len(browser.tabs) == expected_number_of_tabs, 10)
      except:
        logging.error('Number of tabs is %s' % len(browser.tabs))
        raise
      self.assertEquals(expected_number_of_tabs, len(browser.tabs))

  @classmethod
  def tearDownClass(cls):
    shutil.rmtree(cls._profile_dir)

class TestBrowserOperationDoNotLeakTempFiles(unittest.TestCase):

  @decorators.Enabled('win', 'mac', 'linux')
  @decorators.Isolated
  def testBrowserNotLeakingTempFiles(self):
    options = options_for_unittests.GetCopy()
    browser_to_create = browser_finder.FindBrowser(options)
    self.assertIsNotNone(browser_to_create)
    before_browser_run_temp_dir_content = os.listdir(tempfile.tempdir)
    with browser_to_create.Create(options) as browser:
      tab = browser.tabs.New()
      tab.Navigate('about:blank')
      self.assertEquals(2, tab.EvaluateJavaScript('1 + 1'))
    after_browser_run_temp_dir_content = os.listdir(tempfile.tempdir)
    self.assertEqual(before_browser_run_temp_dir_content,
                     after_browser_run_temp_dir_content)
<commit_msg>[Gardening] Disable failing unit test<commit_after># Copyright 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import logging
import os
import shutil
import tempfile
import unittest

from telemetry.core import util
from telemetry import decorators
from telemetry.internal.browser import browser as browser_module
from telemetry.internal.browser import browser_finder
from telemetry.internal.platform import gpu_device
from telemetry.internal.platform import gpu_info
from telemetry.internal.platform import system_info
from telemetry.internal.util import path
from telemetry.testing import browser_test_case
from telemetry.testing import options_for_unittests
from telemetry.timeline import tracing_config

import mock

class IntentionalException(Exception):
  pass

class BrowserTest(browser_test_case.BrowserTestCase):
  def testBrowserCreation(self):
    self.assertEquals(1, len(self._browser.tabs))

    # Different browsers boot up to different things.
    assert self._browser.tabs[0].url

  @decorators.Enabled('has tabs')
  def testNewCloseTab(self):
    existing_tab = self._browser.tabs[0]
    self.assertEquals(1, len(self._browser.tabs))
    existing_tab_url = existing_tab.url

    new_tab = self._browser.tabs.New()
    self.assertEquals(2, len(self._browser.tabs))
    self.assertEquals(existing_tab.url, existing_tab_url)
    self.assertEquals(new_tab.url, 'about:blank')

    new_tab.Close()
    self.assertEquals(1, len(self._browser.tabs))
    self.assertEquals(existing_tab.url, existing_tab_url)

  def testMultipleTabCalls(self):
    self._browser.tabs[0].Navigate(self.UrlOfUnittestFile('blank.html'))
    self._browser.tabs[0].WaitForDocumentReadyStateToBeInteractiveOrBetter()

  def testTabCallByReference(self):
    tab = self._browser.tabs[0]
    tab.Navigate(self.UrlOfUnittestFile('blank.html'))
    self._browser.tabs[0].WaitForDocumentReadyStateToBeInteractiveOrBetter()

  # Disable bug: https://github.com/catapult-project/catapult/issues/2455
  @decorators.Enabled('has tabs')
  @decorators.Disabled('linux', 'mac')
  def testCloseReferencedTab(self):
    self._browser.tabs.New()
    tab = self._browser.tabs[0]
    tab.Navigate(self.UrlOfUnittestFile('blank.html'))
    tab.Close()
    self.assertEquals(1, len(self._browser.tabs))

  @decorators.Enabled('has tabs')
  def testForegroundTab(self):
    # Should be only one tab at this stage, so that must be the foreground tab
    original_tab = self._browser.tabs[0]
    self.assertEqual(self._browser.foreground_tab, original_tab)
    new_tab = self._browser.tabs.New()
    # New tab shouls be foreground tab
    self.assertEqual(self._browser.foreground_tab, new_tab)
    # Make sure that activating the background tab makes it the foreground tab
    original_tab.Activate()
    self.assertEqual(self._browser.foreground_tab, original_tab)
    # Closing the current foreground tab should switch the foreground tab to the
    # other tab
    original_tab.Close()
    self.assertEqual(self._browser.foreground_tab, new_tab)

  def testGetSystemInfo(self):
    if not self._browser.supports_system_info:
      logging.warning(
          'Browser does not support getting system info, skipping test.')
      return

    info = self._browser.GetSystemInfo()

    self.assertTrue(isinstance(info, system_info.SystemInfo))
    self.assertTrue(hasattr(info, 'model_name'))
    self.assertTrue(hasattr(info, 'gpu'))
    self.assertTrue(isinstance(info.gpu, gpu_info.GPUInfo))
    self.assertTrue(hasattr(info.gpu, 'devices'))
    self.assertTrue(len(info.gpu.devices) > 0)
    for g in info.gpu.devices:
      self.assertTrue(isinstance(g, gpu_device.GPUDevice))

  def testGetSystemInfoNotCachedObject(self):
    if not self._browser.supports_system_info:
      logging.warning(
          'Browser does not support getting system info, skipping test.')
      return

    info_a = self._browser.GetSystemInfo()
    info_b = self._browser.GetSystemInfo()
    self.assertFalse(info_a is info_b)

  def testGetSystemTotalMemory(self):
    self.assertTrue(self._browser.memory_stats['SystemTotalPhysicalMemory'] > 0)

  @decorators.Disabled('cros-chrome-guest', 'system-guest')  # chromeos guest
  def testIsTracingRunning(self):
    tracing_controller = self._browser.platform.tracing_controller
    if not tracing_controller.IsChromeTracingSupported():
      return
    self.assertFalse(tracing_controller.is_tracing_running)
    config = tracing_config.TracingConfig()
    config.enable_chrome_trace = True
    tracing_controller.StartTracing(config)
    self.assertTrue(tracing_controller.is_tracing_running)
    tracing_controller.StopTracing()
    self.assertFalse(tracing_controller.is_tracing_running)

class CommandLineBrowserTest(browser_test_case.BrowserTestCase):
  @classmethod
  def CustomizeBrowserOptions(cls, options):
    options.AppendExtraBrowserArgs('--user-agent=telemetry')

  def testCommandLineOverriding(self):
    # This test starts the browser with --user-agent=telemetry. This tests
    # whether the user agent is then set.
    t = self._browser.tabs[0]
    t.Navigate(self.UrlOfUnittestFile('blank.html'))
    t.WaitForDocumentReadyStateToBeInteractiveOrBetter()
    self.assertEquals(t.EvaluateJavaScript('navigator.userAgent'),
                      'telemetry')

class DirtyProfileBrowserTest(browser_test_case.BrowserTestCase):
  @classmethod
  def CustomizeBrowserOptions(cls, options):
    options.profile_type = 'small_profile'

  @decorators.Disabled('chromeos')  # crbug.com/243912
  def testDirtyProfileCreation(self):
    self.assertEquals(1, len(self._browser.tabs))

class BrowserLoggingTest(browser_test_case.BrowserTestCase):
  @classmethod
  def CustomizeBrowserOptions(cls, options):
    options.logging_verbosity = options.VERBOSE_LOGGING

  @decorators.Disabled('chromeos', 'android')
  def testLogFileExist(self):
    self.assertTrue(
       os.path.isfile(self._browser._browser_backend.log_file_path))

def _GenerateBrowserProfile(number_of_tabs):
  """ Generate a browser profile which browser had |number_of_tabs| number of
  tabs opened before it was closed.
      Returns:
        profile_dir: the directory of profile.
  """
  profile_dir = tempfile.mkdtemp()
  options = options_for_unittests.GetCopy()
  options.browser_options.output_profile_path = profile_dir
  browser_to_create = browser_finder.FindBrowser(options)
  with browser_to_create.Create(options) as browser:
    browser.platform.SetHTTPServerDirectories(path.GetUnittestDataDir())
    blank_file_path = os.path.join(path.GetUnittestDataDir(), 'blank.html')
    blank_url = browser.platform.http_server.UrlOf(blank_file_path)
    browser.foreground_tab.Navigate(blank_url)
    browser.foreground_tab.WaitForDocumentReadyStateToBeComplete()
    for _ in xrange(number_of_tabs - 1):
      tab = browser.tabs.New()
      tab.Navigate(blank_url)
      tab.WaitForDocumentReadyStateToBeComplete()
  return profile_dir

class BrowserCreationTest(unittest.TestCase):
  def setUp(self):
    self.mock_browser_backend = mock.MagicMock()
    self.mock_platform_backend = mock.MagicMock()

  def testCleanedUpCalledWhenExceptionRaisedInBrowserCreation(self):
    self.mock_platform_backend.platform.FlushDnsCache.side_effect = (
        IntentionalException('Boom!'))
    with self.assertRaises(IntentionalException):
      browser_module.Browser(
         self.mock_browser_backend, self.mock_platform_backend,
         credentials_path=None)
    self.assertTrue(self.mock_platform_backend.WillCloseBrowser.called)

  def testOriginalExceptionNotSwallow(self):
    self.mock_platform_backend.platform.FlushDnsCache.side_effect = (
        IntentionalException('Boom!'))
    self.mock_platform_backend.WillCloseBrowser.side_effect = (
        IntentionalException('Cannot close browser!'))
    with self.assertRaises(IntentionalException) as context:
      browser_module.Browser(
         self.mock_browser_backend, self.mock_platform_backend,
         credentials_path=None)
    self.assertIn('Boom!', context.exception.message)

class BrowserRestoreSessionTest(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    cls._number_of_tabs = 4
    cls._profile_dir = _GenerateBrowserProfile(cls._number_of_tabs)
    cls._options = options_for_unittests.GetCopy()
    cls._options.browser_options.AppendExtraBrowserArgs(
        ['--restore-last-session'])
    cls._options.browser_options.profile_dir = cls._profile_dir
    cls._browser_to_create = browser_finder.FindBrowser(cls._options)

  @decorators.Enabled('has tabs')
  @decorators.Disabled('chromeos', 'win', 'mac')
  # TODO(nednguyen): Enable this test on windowsn platform
  def testRestoreBrowserWithMultipleTabs(self):
    with self._browser_to_create.Create(self._options) as browser:
      # The number of tabs will be self._number_of_tabs + 1 as it includes the
      # old tabs and a new blank tab.
      expected_number_of_tabs = self._number_of_tabs + 1
      try:
        util.WaitFor(lambda: len(browser.tabs) == expected_number_of_tabs, 10)
      except:
        logging.error('Number of tabs is %s' % len(browser.tabs))
        raise
      self.assertEquals(expected_number_of_tabs, len(browser.tabs))

  @classmethod
  def tearDownClass(cls):
    shutil.rmtree(cls._profile_dir)

class TestBrowserOperationDoNotLeakTempFiles(unittest.TestCase):

  @decorators.Enabled('win', 'mac', 'linux')
  @decorators.Isolated
  def testBrowserNotLeakingTempFiles(self):
    options = options_for_unittests.GetCopy()
    browser_to_create = browser_finder.FindBrowser(options)
    self.assertIsNotNone(browser_to_create)
    before_browser_run_temp_dir_content = os.listdir(tempfile.tempdir)
    with browser_to_create.Create(options) as browser:
      tab = browser.tabs.New()
      tab.Navigate('about:blank')
      self.assertEquals(2, tab.EvaluateJavaScript('1 + 1'))
    after_browser_run_temp_dir_content = os.listdir(tempfile.tempdir)
    self.assertEqual(before_browser_run_temp_dir_content,
                     after_browser_run_temp_dir_content)
<|endoftext|>


<commit_before><commit_msg>Rm noises indexing debug messages<commit_after><|endoftext|>


<commit_before>def _all(iterable):
    for elem in iterable:
        if not elem:
            return False
    return True

try:
    all = all
except NameError:
    all = _all<commit_msg>Cosmetic tweak.<commit_after>def _all(iterable):
    for elem in iterable:
        if not elem:
            return False
    return True

try:
    all = all
except NameError:
    all = _all
<|endoftext|>


<commit_before>from busbus import util

import itertools

class Queryable(util.Iterable):

    def __init__(self, it, query_funcs=None):
        self.it = iter(it)
        self.query_funcs = tuple(query_funcs) if query_funcs else ()

    def __next__(self):
        while True:
            value = next(self.it)
            if all(query_func(value) for query_func in self.query_funcs):
                return value

    def where(self, query_func=None, **kwargs):
        if not query_func and not kwargs:
            return self
        new_funcs = [query_func] if query_func else []
        for k, v in kwargs.items():
            new_funcs.append(lambda obj: getattr(obj, k) == v)
        return Queryable(self.it, self.query_funcs + tuple(new_funcs))

    @staticmethod
    def chain(*its):
        return ChainedQueryable(*its)

class ChainedQueryable(Queryable):

    def __init__(self, *its):
        self.its = its
        self.it = itertools.chain(*its)

    def __next__(self):
        return next(self.it)

    def where(self, query_func=None, **kwargs):
        its = []
        for it in self.its:
            its.append(it.where(query_func, **kwargs))
        return ChainedQueryable(*its)
<commit_msg>Don't throw AttributeError in Queryables if we can avoid it<commit_after>from busbus import util

import itertools

class Queryable(util.Iterable):

    def __init__(self, it, query_funcs=None):
        self.it = iter(it)
        self.query_funcs = tuple(query_funcs) if query_funcs else ()

    def __next__(self):
        while True:
            value = next(self.it)
            if all(query_func(value) for query_func in self.query_funcs):
                return value

    def where(self, query_func=None, **kwargs):
        if not query_func and not kwargs:
            return self
        new_funcs = [query_func] if query_func else []
        for k, v in kwargs.items():
            new_funcs.append(lambda obj: (getattr(obj, k) == v
                                          if hasattr(obj, k) else False))
        return Queryable(self.it, self.query_funcs + tuple(new_funcs))

    @staticmethod
    def chain(*its):
        return ChainedQueryable(*its)

class ChainedQueryable(Queryable):

    def __init__(self, *its):
        self.its = its
        self.it = itertools.chain(*its)

    def __next__(self):
        return next(self.it)

    def where(self, query_func=None, **kwargs):
        its = []
        for it in self.its:
            its.append(it.where(query_func, **kwargs))
        return ChainedQueryable(*its)
<|endoftext|>