<commit_before>from elasticsearch import Elasticsearch

from enum import Enum
from collections import defaultdict

from api.apiutils import Hit
import config as c

class KWType(Enum):
    KW_TEXT = 0
    KW_SCHEMA = 1
    KW_ENTITIES = 2

class StoreHandler:

    # Store client
    client = None

    def __init__(self):
        """
            Uses the configuration file to create a connection to the store
            :return:
            """
        global client
        client = Elasticsearch([{'host': c.db_host, 'port': c.db_port}])

    def close(self):
        print("TODO")

    def get_all_fields(self):
        """
        Reads all fields, described as (id, source_name, field_name) from the store.
        :return: a list of all fields with the form (id, source_name, field_name)
        """
        body = {"query": {"match_all": {}}}
        res = client.search(index='profile', body=body, scroll="10m",
                            filter_path=['_scroll_id',
                                         'hits.hits._id',
                                         'hits.total',
                                         'hits.hits._source.dbName',
                                         'hits.hits._source.sourceName',
                                         'hits.hits._source.columnName',
                                         'hits.hits._source.totalValues',
                                         'hits.hits._source.uniqueValues']
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                id_source_and_file_name = (h['_id'], h['_source']['dbName'], h['_source']['sourceName'],
                                           h['_source']['columnName'], h['_source']['totalValues'],
                                           h['_source']['uniqueValues'])
                yield id_source_and_file_name
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.hits._source.dbName',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName',
                                             'hits.hits._source.totalValues',
                                             'hits.hits._source.uniqueValues']
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)

    def get_all_fields_with(self, attrs):
        # FIXME: this function was not updated after 2 refactoring processes.
        """
        Reads all fields, described as (id, source_name, field_name) from the store.
        :return: a list of all fields with the form (id, source_name, field_name)
        """
        template = 'hits.hits._source.'
        filter_path = ['_scroll_id',
                       'hits.hits._id',
                       'hits.total',
                       'hits.hits._source.dbName',
                       'hits.hits._source.sourceName',
                       'hits.hits._source.columnName']
        for attr in attrs:
            new_filter_path = template + attr
            filter_path.append(new_filter_path)

        body = {"query": {"match_all": {}}}
        res = client.search(index='profile', body=body, scroll="10m",
                            filter_path=filter_path
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                toret = []
                toret.append(h['_id'])
                toret.append(h['_source']['sourceName'])
                toret.append(h['_source']['columnName'])
                for attr in attrs:
                    toret.append(h['_source'][attr])
                tuple_result = tuple(toret)
                yield tuple_result
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=filter_path
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)

    def peek_values(self, field, num_values):
        """
        Reads sample values for the given field
        :param field: The field from which to read values
        :param num_values: The number of values to read
        :return: A list with the sample values read for field
        """
        print("TODO")

    def search_keywords(self, keywords, elasticfieldname, max_hits=15):
        """
        Performs a search query on elastic_field_name to match the provided keywords
        :param keywords: the list of keyword to match
        :param elasticfieldname: what is the field in the store where to apply the query
        :return: the list of documents that contain the keywords
        """
        index = None
        query_body = None
        filter_path = ['hits.hits._source.id',
                       'hits.hits._score',
                       'hits.total',
                       'hits.hits._source.dbName',
                       'hits.hits._source.sourceName',
                       'hits.hits._source.columnName']
        if elasticfieldname == KWType.KW_TEXT:
            index = "text"
            query_body = {"from": 0, "size": max_hits,
                          "query": {"match": {"text": keywords}}}
        elif elasticfieldname == KWType.KW_SCHEMA:
            index = "profile"
            query_body = {"from": 0, "size": max_hits,
                          "query": {"match": {"columnName": keywords}}}
        elif elasticfieldname == KWType.KW_ENTITIES:
            index = "profile"
            query_body = {"from": 0, "size": max_hits,
                          "query": {"match": {"entities": keywords}}}
        res = client.search(index=index, body=query_body,
                            filter_path=filter_path)
        if res['hits']['total'] == 0:
            return []
        for el in res['hits']['hits']:
            data = Hit(el['_source']['id'], el['_source']['dbName'], el['_source']['sourceName'],
                       el['_source']['columnName'], el['_score'])
            yield data

    def get_all_fields_entities(self):
        """
        Retrieves all fields and entities from the store
        :return: (fields, entities)
        """
        results = self.get_all_fields_with(['entities'])
        fields = []
        ents = []
        for r in results:
            (nid, sn, fn, entities) = r
            fields.append((nid, sn, fn))
            ents.append(entities)
        return fields, ents

    def get_all_docs_from_text_with_idx_id(self, doc_id):
        """
        Reads all fields, described as (id, source_name, field_name) from the store.
        :return: a list of all fields with the form (id, source_name, field_name)
        """
        body = {"query": {"bool": {"must": [{"match": {"id": doc_id}}]}}}
        res = client.search(index='text', body=body, scroll="10m",
                            filter_path=['_scroll_id',
                                         'hits.hits._id',
                                         'hits.total'
                                         ]
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                raw_id_doc = h['_id']
                yield raw_id_doc
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.hits._source.id']
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)

    def get_all_fields_text_signatures(self, network):
        def partition_ids(ids, partition_size=50):
            for i in range(0, len(ids), partition_size):
                yield ids[i:i + partition_size]

        def filter_term_vector_by_frequency(term_dict):
            # FIXME: add filter by term length
            filtered = []
            for k, v in term_dict.items():
                if len(k) > 2:
                    if v > 5:
                        filtered.append(k)
            return filtered

        text_signatures = []
        total = 0
        for nid in network.iterate_ids():
            total += 1
            print("text_sig: " + str(total))
            # We retrieve all documents indexed with the same id in 'text'
            docs = self.get_all_docs_from_text_with_idx_id(nid)
            ids = [x for x in docs]
            # partition ids so that they fit in one http request
            ids_partitions = partition_ids(ids)
            all_terms = defaultdict(int)
            for partition in ids_partitions:
                # We get the term vectors for each group of those documents
                # , body=term_body)
                ans = client.mtermvectors(
                    index='text', ids=partition, doc_type='column')
                # We merge them somehow
                found_docs = ans['docs']
                for doc in found_docs:
                    term_vectors = doc['term_vectors']
                    if 'text' in term_vectors:
                        # terms = list(term_vectors['text']['terms'].keys())
                        terms_and_freq = term_vectors['text']['terms']
                        for term, freq_dict in terms_and_freq.items():
                            # we don't care about the value
                            all_terms[term] = all_terms[
                                                  term] + freq_dict['term_freq']
            filtered_term_vector = filter_term_vector_by_frequency(all_terms)
            if len(filtered_term_vector) > 0:
                data = (nid, filtered_term_vector)
                text_signatures.append(data)
        return text_signatures

    def get_all_fields_num_signatures(self):
        """
        Retrieves numerical fields and signatures from the store
        :return: (fields, numsignatures)
        """
        query_body = {
            "query": {"bool": {"filter": [{"term": {"dataType": "N"}}]}}}
        res = client.search(index='profile', body=query_body, scroll="10m",
                            filter_path=['_scroll_id',
                                         'hits.hits._id',
                                         'hits.total',
                                         'hits.hits._source.median',
                                         'hits.hits._source.iqr']
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']

        id_sig = []
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                data = (h['_id'], (h['_source']['median'], h['_source']['iqr']))
                id_sig.append(data)
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.hits._source.median',
                                             'hits.hits._source.iqr']
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)
        return id_sig

if __name__ == "__main__":
    print("Elastic Store")

    """
        def get_all_text_fields(self):
            query_body = {
                "query": {"bool": {"filter": [{"term": {"dataType": "T"}}]}}}
            res = client.search(index='profile', body=query_body, scroll="10m",
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.total',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName',
                                             'hits.hits._source.totalValues',
                                             'hits.hits._source.uniqueValues']
                                )
            scroll_id = res['_scroll_id']
            remaining = res['hits']['total']
            while remaining > 0:
                hits = res['hits']['hits']
                for h in hits:
                    id_source_and_file_name = (h['_id'], h['_source']['sourceName'], h['_source']['columnName'],
                                               h['_source']['totalValues'], h['_source']['uniqueValues'])
                    yield id_source_and_file_name
                    remaining -= 1
                res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                    filter_path=['_scroll_id',
                                                 'hits.hits._id',
                                                 'hits.hits._source.sourceName',
                                                 'hits.hits._source.columnName',
                                                 'hits.hits._source.totalValues',
                                                 'hits.hits._source.uniqueValues']
                                    )
                scroll_id = res['_scroll_id']  # update the scroll_id
            client.clear_scroll(scroll_id=scroll_id)

        def get_fields_text_index(self):
            '''
            Reads all fields, described as (id, source_name, field_name) from the store (text index).
            :return: a list of all fields with the form (id, source_name, field_name)
            '''
            body = {"query": {"match_all": {}}}
            res = client.search(index='text', body=body, scroll="10m",
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.total',
                                             'hits.hits._source.id',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName']
                                )
            scroll_id = res['_scroll_id']
            remaining = res['hits']['total']
            while remaining > 0:
                hits = res['hits']['hits']
                for h in hits:
                    rawid_id_source_and_file_name = (h['_id'], h['_source']['id'],
                                                     h['_source']['sourceName'], h['_source']['columnName'])
                    yield rawid_id_source_and_file_name
                    remaining -= 1
                res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                    filter_path=['_scroll_id',
                                                 'hits.hits._id',
                                                 'hits.hits._source.id',
                                                 'hits.hits._source.sourceName',
                                                 'hits.hits._source.columnName']
                                    )
                scroll_id = res['_scroll_id']  # update the scroll_id
            client.clear_scroll(scroll_id=scroll_id)

        def get_all_fields_of_source(self, source_name):
            body = {"query": {"match": {"sourceName": source_name}}}
            res = client.search(index='profile', body=body, scroll="10m",
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.total',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName']
                                )
            scroll_id = res['_scroll_id']
            remaining = res['hits']['total']
            while remaining > 0:
                hits = res['hits']['hits']
                for h in hits:
                    hit = Hit(h['_id'], h['_source']['sourceName'],
                              h['_source']['columnName'], -1)
                    yield hit
                    remaining -= 1
                res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                    filter_path=['_scroll_id',
                                                 'hits.hits._id',
                                                 'hits.hits._source.sourceName',
                                                 'hits.hits._source.columnName']
                                    )
                scroll_id = res['_scroll_id']  # update the scroll_id
            client.clear_scroll(scroll_id=scroll_id)
        """
<commit_msg>Added new filtering criteria for making tf-idf more accurate<commit_after>import re
from elasticsearch import Elasticsearch

from enum import Enum
from collections import defaultdict

from api.apiutils import Hit
import config as c

class KWType(Enum):
    KW_TEXT = 0
    KW_SCHEMA = 1
    KW_ENTITIES = 2

class StoreHandler:

    # Store client
    client = None

    def __init__(self):
        """
            Uses the configuration file to create a connection to the store
            :return:
            """
        global client
        client = Elasticsearch([{'host': c.db_host, 'port': c.db_port}])

    def close(self):
        print("TODO")

    def get_all_fields(self):
        """
        Reads all fields, described as (id, source_name, field_name) from the store.
        :return: a list of all fields with the form (id, source_name, field_name)
        """
        body = {"query": {"match_all": {}}}
        res = client.search(index='profile', body=body, scroll="10m",
                            filter_path=['_scroll_id',
                                         'hits.hits._id',
                                         'hits.total',
                                         'hits.hits._source.dbName',
                                         'hits.hits._source.sourceName',
                                         'hits.hits._source.columnName',
                                         'hits.hits._source.totalValues',
                                         'hits.hits._source.uniqueValues']
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                id_source_and_file_name = (h['_id'], h['_source']['dbName'], h['_source']['sourceName'],
                                           h['_source']['columnName'], h['_source']['totalValues'],
                                           h['_source']['uniqueValues'])
                yield id_source_and_file_name
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.hits._source.dbName',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName',
                                             'hits.hits._source.totalValues',
                                             'hits.hits._source.uniqueValues']
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)

    def get_all_fields_with(self, attrs):
        # FIXME: this function was not updated after 2 refactoring processes.
        """
        Reads all fields, described as (id, source_name, field_name) from the store.
        :return: a list of all fields with the form (id, source_name, field_name)
        """
        template = 'hits.hits._source.'
        filter_path = ['_scroll_id',
                       'hits.hits._id',
                       'hits.total',
                       'hits.hits._source.dbName',
                       'hits.hits._source.sourceName',
                       'hits.hits._source.columnName']
        for attr in attrs:
            new_filter_path = template + attr
            filter_path.append(new_filter_path)

        body = {"query": {"match_all": {}}}
        res = client.search(index='profile', body=body, scroll="10m",
                            filter_path=filter_path
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                toret = []
                toret.append(h['_id'])
                toret.append(h['_source']['sourceName'])
                toret.append(h['_source']['columnName'])
                for attr in attrs:
                    toret.append(h['_source'][attr])
                tuple_result = tuple(toret)
                yield tuple_result
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=filter_path
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)

    def peek_values(self, field, num_values):
        """
        Reads sample values for the given field
        :param field: The field from which to read values
        :param num_values: The number of values to read
        :return: A list with the sample values read for field
        """
        print("TODO")

    def search_keywords(self, keywords, elasticfieldname, max_hits=15):
        """
        Performs a search query on elastic_field_name to match the provided keywords
        :param keywords: the list of keyword to match
        :param elasticfieldname: what is the field in the store where to apply the query
        :return: the list of documents that contain the keywords
        """
        index = None
        query_body = None
        filter_path = ['hits.hits._source.id',
                       'hits.hits._score',
                       'hits.total',
                       'hits.hits._source.dbName',
                       'hits.hits._source.sourceName',
                       'hits.hits._source.columnName']
        if elasticfieldname == KWType.KW_TEXT:
            index = "text"
            query_body = {"from": 0, "size": max_hits,
                          "query": {"match": {"text": keywords}}}
        elif elasticfieldname == KWType.KW_SCHEMA:
            index = "profile"
            query_body = {"from": 0, "size": max_hits,
                          "query": {"match": {"columnName": keywords}}}
        elif elasticfieldname == KWType.KW_ENTITIES:
            index = "profile"
            query_body = {"from": 0, "size": max_hits,
                          "query": {"match": {"entities": keywords}}}
        res = client.search(index=index, body=query_body,
                            filter_path=filter_path)
        if res['hits']['total'] == 0:
            return []
        for el in res['hits']['hits']:
            data = Hit(el['_source']['id'], el['_source']['dbName'], el['_source']['sourceName'],
                       el['_source']['columnName'], el['_score'])
            yield data

    def get_all_fields_entities(self):
        """
        Retrieves all fields and entities from the store
        :return: (fields, entities)
        """
        results = self.get_all_fields_with(['entities'])
        fields = []
        ents = []
        for r in results:
            (nid, sn, fn, entities) = r
            fields.append((nid, sn, fn))
            ents.append(entities)
        return fields, ents

    def get_all_docs_from_text_with_idx_id(self, doc_id):
        """
        Reads all fields, described as (id, source_name, field_name) from the store.
        :return: a list of all fields with the form (id, source_name, field_name)
        """
        body = {"query": {"bool": {"must": [{"match": {"id": doc_id}}]}}}
        res = client.search(index='text', body=body, scroll="10m",
                            filter_path=['_scroll_id',
                                         'hits.hits._id',
                                         'hits.total'
                                         ]
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                raw_id_doc = h['_id']
                yield raw_id_doc
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.hits._source.id']
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)

    def get_all_fields_text_signatures(self, network):
        def partition_ids(ids, partition_size=50):
            for i in range(0, len(ids), partition_size):
                yield ids[i:i + partition_size]

        def filter_term_vector_by_frequency(term_dict):
            # FIXME: add filter by term length
            filtered = []
            for k, v in term_dict.items():
                if len(k) > 2:
                    if v > 3:
                        try:
                            float(k)
                            continue
                        except ValueError:
                            matches = re.findall('[0-9]', k)
                            if len(matches) == 0:
                                filtered.append(k)
            return filtered

        text_signatures = []
        total = 0
        for nid in network.iterate_ids():
            total += 1
            print("text_sig: " + str(total))
            # We retrieve all documents indexed with the same id in 'text'
            docs = self.get_all_docs_from_text_with_idx_id(nid)
            ids = [x for x in docs]
            # partition ids so that they fit in one http request
            ids_partitions = partition_ids(ids)
            all_terms = defaultdict(int)
            for partition in ids_partitions:
                # We get the term vectors for each group of those documents
                # , body=term_body)
                ans = client.mtermvectors(
                    index='text', ids=partition, doc_type='column')
                # We merge them somehow
                found_docs = ans['docs']
                for doc in found_docs:
                    term_vectors = doc['term_vectors']
                    if 'text' in term_vectors:
                        # terms = list(term_vectors['text']['terms'].keys())
                        terms_and_freq = term_vectors['text']['terms']
                        for term, freq_dict in terms_and_freq.items():
                            # we don't care about the value
                            all_terms[term] = all_terms[
                                                  term] + freq_dict['term_freq']
            filtered_term_vector = filter_term_vector_by_frequency(all_terms)
            if len(filtered_term_vector) > 0:
                data = (nid, filtered_term_vector)
                text_signatures.append(data)
        return text_signatures

    def get_all_fields_num_signatures(self):
        """
        Retrieves numerical fields and signatures from the store
        :return: (fields, numsignatures)
        """
        query_body = {
            "query": {"bool": {"filter": [{"term": {"dataType": "N"}}]}}}
        res = client.search(index='profile', body=query_body, scroll="10m",
                            filter_path=['_scroll_id',
                                         'hits.hits._id',
                                         'hits.total',
                                         'hits.hits._source.median',
                                         'hits.hits._source.iqr']
                            )
        scroll_id = res['_scroll_id']
        remaining = res['hits']['total']

        id_sig = []
        while remaining > 0:
            hits = res['hits']['hits']
            for h in hits:
                data = (h['_id'], (h['_source']['median'], h['_source']['iqr']))
                id_sig.append(data)
                remaining -= 1
            res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.hits._source.median',
                                             'hits.hits._source.iqr']
                                )
            scroll_id = res['_scroll_id']  # update the scroll_id
        client.clear_scroll(scroll_id=scroll_id)
        return id_sig

if __name__ == "__main__":
    print("Elastic Store")

    """
        def get_all_text_fields(self):
            query_body = {
                "query": {"bool": {"filter": [{"term": {"dataType": "T"}}]}}}
            res = client.search(index='profile', body=query_body, scroll="10m",
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.total',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName',
                                             'hits.hits._source.totalValues',
                                             'hits.hits._source.uniqueValues']
                                )
            scroll_id = res['_scroll_id']
            remaining = res['hits']['total']
            while remaining > 0:
                hits = res['hits']['hits']
                for h in hits:
                    id_source_and_file_name = (h['_id'], h['_source']['sourceName'], h['_source']['columnName'],
                                               h['_source']['totalValues'], h['_source']['uniqueValues'])
                    yield id_source_and_file_name
                    remaining -= 1
                res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                    filter_path=['_scroll_id',
                                                 'hits.hits._id',
                                                 'hits.hits._source.sourceName',
                                                 'hits.hits._source.columnName',
                                                 'hits.hits._source.totalValues',
                                                 'hits.hits._source.uniqueValues']
                                    )
                scroll_id = res['_scroll_id']  # update the scroll_id
            client.clear_scroll(scroll_id=scroll_id)

        def get_fields_text_index(self):
            '''
            Reads all fields, described as (id, source_name, field_name) from the store (text index).
            :return: a list of all fields with the form (id, source_name, field_name)
            '''
            body = {"query": {"match_all": {}}}
            res = client.search(index='text', body=body, scroll="10m",
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.total',
                                             'hits.hits._source.id',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName']
                                )
            scroll_id = res['_scroll_id']
            remaining = res['hits']['total']
            while remaining > 0:
                hits = res['hits']['hits']
                for h in hits:
                    rawid_id_source_and_file_name = (h['_id'], h['_source']['id'],
                                                     h['_source']['sourceName'], h['_source']['columnName'])
                    yield rawid_id_source_and_file_name
                    remaining -= 1
                res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                    filter_path=['_scroll_id',
                                                 'hits.hits._id',
                                                 'hits.hits._source.id',
                                                 'hits.hits._source.sourceName',
                                                 'hits.hits._source.columnName']
                                    )
                scroll_id = res['_scroll_id']  # update the scroll_id
            client.clear_scroll(scroll_id=scroll_id)

        def get_all_fields_of_source(self, source_name):
            body = {"query": {"match": {"sourceName": source_name}}}
            res = client.search(index='profile', body=body, scroll="10m",
                                filter_path=['_scroll_id',
                                             'hits.hits._id',
                                             'hits.total',
                                             'hits.hits._source.sourceName',
                                             'hits.hits._source.columnName']
                                )
            scroll_id = res['_scroll_id']
            remaining = res['hits']['total']
            while remaining > 0:
                hits = res['hits']['hits']
                for h in hits:
                    hit = Hit(h['_id'], h['_source']['sourceName'],
                              h['_source']['columnName'], -1)
                    yield hit
                    remaining -= 1
                res = client.scroll(scroll="3m", scroll_id=scroll_id,
                                    filter_path=['_scroll_id',
                                                 'hits.hits._id',
                                                 'hits.hits._source.sourceName',
                                                 'hits.hits._source.columnName']
                                    )
                scroll_id = res['_scroll_id']  # update the scroll_id
            client.clear_scroll(scroll_id=scroll_id)
        """
<|endoftext|>


<commit_before>{
  'variables': {
    # The libraries brightray will be compiled to.
    'linux_system_libraries': 'gtk+-2.0 dbus-1 x11 xi xcursor xdamage xrandr xcomposite xext xfixes xrender xtst xscrnsaver gconf-2.0 gmodule-2.0 nss'
  },
  'includes': [
    'filenames.gypi',
  ],
  'targets': [
    {
      'target_name': 'brightray',
      'type': 'static_library',
      'include_dirs': [
        '.',
        '<(libchromiumcontent_src_dir)',
        '<(libchromiumcontent_src_dir)/skia/config',
        '<(libchromiumcontent_src_dir)/third_party/skia/include/core',
        '<(libchromiumcontent_src_dir)/third_party/mojo/src',
        '<(libchromiumcontent_src_dir)/third_party/WebKit',
        '<(libchromiumcontent_dir)/gen',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          '.',
          '..',
          '<(libchromiumcontent_src_dir)',
          '<(libchromiumcontent_src_dir)/gpu',
          '<(libchromiumcontent_src_dir)/skia/config',
          '<(libchromiumcontent_src_dir)/third_party/skia/include/core',
          '<(libchromiumcontent_src_dir)/third_party/skia/include/config',
          '<(libchromiumcontent_src_dir)/third_party/icu/source/common',
          '<(libchromiumcontent_src_dir)/third_party/mojo/src',
          '<(libchromiumcontent_src_dir)/third_party/khronos',
          '<(libchromiumcontent_src_dir)/third_party/WebKit',
          '<(libchromiumcontent_dir)/gen',
        ],
      },
      'sources': [ '<@(brightray_sources)' ],
      'conditions': [
        # Link with libraries of libchromiumcontent.
        ['OS=="linux" and libchromiumcontent_component==0', {
          # On Linux we have to use "--whole-archive" to force executable
          # to include all symbols, otherwise we will have plenty of
          # unresolved symbols errors.
          'direct_dependent_settings': {
            'ldflags': [
              '-Wl,--whole-archive',
              '<@(libchromiumcontent_libraries)',
              '-Wl,--no-whole-archive',
            ],
          }
        }, {  # (Release build on Linux)
          'link_settings': {
            'libraries': [ '<@(libchromiumcontent_libraries)' ]
          },
        }],  # (Normal builds)
        # Linux specific link settings.
        ['OS=="linux"', {
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other <(linux_system_libraries))',
            ],
            'libraries': [
              '-lpthread',
              '<!@(<(pkg-config) --libs-only-l <(linux_system_libraries))',
            ],
          },
          'cflags': [
            '<!@(<(pkg-config) --cflags <(linux_system_libraries))',
            # Needed by using libgtk2ui:
            '-Wno-deprecated-register',
            '-Wno-sentinel',
          ],
          'cflags_cc': [
            '-Wno-reserved-user-defined-literal',
          ],
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags <(linux_system_libraries))',
              '-Wno-deprecated-register',
              '-Wno-sentinel',
            ],
          },
          'conditions': [
            ['libchromiumcontent_component', {
              'link_settings': {
                'libraries': [
                  # Following libraries are always linked statically.
                  '<(libchromiumcontent_dir)/libgtk2ui.a',
                  '<(libchromiumcontent_dir)/libdevtools_discovery.a',
                  '<(libchromiumcontent_dir)/libdevtools_http_handler.a',
                  '<(libchromiumcontent_dir)/libhttp_server.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture_differ_sse2.a',
                  '<(libchromiumcontent_dir)/libsystem_wrappers.a',
                  '<(libchromiumcontent_dir)/librtc_base.a',
                  '<(libchromiumcontent_dir)/librtc_base_approved.a',
                  '<(libchromiumcontent_dir)/libwebrtc_common.a',
                  '<(libchromiumcontent_dir)/libyuv.a',
                  '<(libchromiumcontent_dir)/libcdm_renderer.a',
                  '<(libchromiumcontent_dir)/libsecurity_state.a',
                ],
              },
            }, {
              'link_settings': {
                'libraries': [
                  # Link with ffmpeg.
                  '<(libchromiumcontent_dir)/libffmpeg.so',
                  # Following libraries are required by libchromiumcontent:
                  '-lasound',
                  '-lcap',
                  '-lcups',
                  '-lrt',
                  '-ldl',
                  '-lresolv',
                  '-lfontconfig',
                  '-lfreetype',
                  '-lexpat',
                ],
              },
            }],
            ['target_arch=="arm"', {
              'link_settings': {
                'libraries!': [
                  '<(libchromiumcontent_dir)/libdesktop_capture_differ_sse2.a',
                ],
              },
            }],
          ],
        }],  # OS=="linux"
        ['OS=="mac"', {
          'link_settings': {
            'libraries': [
              '$(SDKROOT)/System/Library/Frameworks/AppKit.framework',
              # Required by webrtc:
              '$(SDKROOT)/System/Library/Frameworks/OpenGL.framework',
              '$(SDKROOT)/System/Library/Frameworks/IOKit.framework',
            ],
          },
          'conditions':  [
            ['libchromiumcontent_component', {
              'link_settings': {
                'libraries': [
                  # Following libraries are always linked statically.
                  '<(libchromiumcontent_dir)/libdevtools_discovery.a',
                  '<(libchromiumcontent_dir)/libdevtools_http_handler.a',
                  '<(libchromiumcontent_dir)/libhttp_server.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture_differ_sse2.a',
                  '<(libchromiumcontent_dir)/librtc_base.a',
                  '<(libchromiumcontent_dir)/librtc_base_approved.a',
                  '<(libchromiumcontent_dir)/libsystem_wrappers.a',
                  '<(libchromiumcontent_dir)/libwebrtc_common.a',
                  '<(libchromiumcontent_dir)/libyuv.a',
                  '<(libchromiumcontent_dir)/libcdm_renderer.a',
                  '<(libchromiumcontent_dir)/libsecurity_state.a',
                ],
              },
            }, {
              'link_settings': {
                'libraries': [
                  # Link with ffmpeg.
                  '<(libchromiumcontent_dir)/libffmpeg.dylib',
                  # Link with system frameworks.
                  # ui_base.gypi:
                  '$(SDKROOT)/System/Library/Frameworks/Accelerate.framework',
                  # net.gypi:
                  '$(SDKROOT)/System/Library/Frameworks/Foundation.framework',
                  '$(SDKROOT)/System/Library/Frameworks/Security.framework',
                  '$(SDKROOT)/System/Library/Frameworks/SystemConfiguration.framework',
                  '-lresolv',
                  # media.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/AudioToolbox.framework',
                  '$(SDKROOT)/System/Library/Frameworks/AudioUnit.framework',
                  '$(SDKROOT)/System/Library/Frameworks/AVFoundation.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreAudio.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreMedia.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreMIDI.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreVideo.framework',
                  # surface.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/IOSurface.framework',
                  # content_common.gypi:
                  '$(SDKROOT)/System/Library/Frameworks/QuartzCore.framework',
                  # base.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/ApplicationServices.framework',
                  '$(SDKROOT)/System/Library/Frameworks/Carbon.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreFoundation.framework',
                  # content_browser.gypi:
                  '-lbsm',
                  # content_common.gypi:
                  '-lsandbox',
                  # bluetooth.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/IOBluetooth.framework',
                ],
              },
            }],
          ]
        }],  # OS=="mac"
        ['OS=="win"', {
          'conditions': [
            ['libchromiumcontent_component', {
              'link_settings': {
                'libraries': [
                  # Needed by desktop_capture.lib:
                  '-ld3d11.lib',
                  # Following libs are always linked statically.
                  '<(libchromiumcontent_dir)/base_static.lib',
                  '<(libchromiumcontent_dir)/sandbox.lib',
                  '<(libchromiumcontent_dir)/sandbox_helper_win.lib',
                  '<(libchromiumcontent_dir)/devtools_discovery.lib',
                  '<(libchromiumcontent_dir)/devtools_http_handler.lib',
                  '<(libchromiumcontent_dir)/http_server.lib',
                  '<(libchromiumcontent_dir)/desktop_capture.lib',
                  '<(libchromiumcontent_dir)/desktop_capture_differ_sse2.lib',
                  '<(libchromiumcontent_dir)/rtc_base.lib',
                  '<(libchromiumcontent_dir)/rtc_base_approved.lib',
                  '<(libchromiumcontent_dir)/system_wrappers.lib',
                  '<(libchromiumcontent_dir)/webrtc_common.lib',
                  '<(libchromiumcontent_dir)/libyuv.lib',
                  '<(libchromiumcontent_dir)/cdm_renderer.lib',
                  '<(libchromiumcontent_dir)/security_state.lib',
                  # Friends of pdf.lib:
                  '<(libchromiumcontent_dir)/pdf.lib',
                  '<(libchromiumcontent_dir)/ppapi_cpp_objects.lib',
                  '<(libchromiumcontent_dir)/ppapi_internal_module.lib',
                  '<(libchromiumcontent_dir)/libjpeg.lib',
                  '<(libchromiumcontent_dir)/pdfium.lib',
                  '<(libchromiumcontent_dir)/fdrm.lib',
                  '<(libchromiumcontent_dir)/formfiller.lib',
                  '<(libchromiumcontent_dir)/fpdfapi.lib',
                  '<(libchromiumcontent_dir)/fpdfdoc.lib',
                  '<(libchromiumcontent_dir)/fpdftext.lib',
                  '<(libchromiumcontent_dir)/fpdftext.lib',
                  '<(libchromiumcontent_dir)/fxcodec.lib',
                  '<(libchromiumcontent_dir)/fxcrt.lib',
                  '<(libchromiumcontent_dir)/fxedit.lib',
                  '<(libchromiumcontent_dir)/fxge.lib',
                  '<(libchromiumcontent_dir)/javascript.lib',
                  '<(libchromiumcontent_dir)/pdfwindow.lib',
                  '<(libchromiumcontent_dir)/bigint.lib',
                  '<(libchromiumcontent_dir)/fx_agg.lib',
                  '<(libchromiumcontent_dir)/fx_freetype.lib',
                  '<(libchromiumcontent_dir)/fx_lcms2.lib',
                  '<(libchromiumcontent_dir)/fx_libopenjpeg.lib',
                  '<(libchromiumcontent_dir)/fx_zlib.lib',
                ],
              },
            }, {
              # Link with system libraries.
              'link_settings': {
                'libraries': [
                  # Link with ffmpeg.
                  '<(libchromiumcontent_dir)/ffmpeg.dll',
                  # content_browser.gypi:
                  '-lsensorsapi.lib',
                  '-lportabledeviceguids.lib',
                  # content_common.gypi:
                  '-ld3d9.lib',
                  '-ld3d11.lib',
                  '-ldxva2.lib',
                  '-lstrmiids.lib',
                  '-lmf.lib',
                  '-lmfplat.lib',
                  '-lmfuuid.lib',
                  # media.gyp:
                  '-ldxguid.lib',
                  '-lmfreadwrite.lib',
                  '-lmfuuid.lib',
                ],
                'msvs_settings': {
                  'VCLinkerTool': {
                    'AdditionalDependencies': [
                      'advapi32.lib',
                      'dbghelp.lib',
                      'delayimp.lib',
                      'dwmapi.lib',
                      'gdi32.lib',
                      'netapi32.lib',
                      'oleacc.lib',
                      'user32.lib',
                      'usp10.lib',
                      'version.lib',
                      'winspool.lib',
                      'wtsapi32.lib',
                      # bluetooth.gyp:
                      'Bthprops.lib',
                      'BluetoothApis.lib',
                      # base.gyp:
                      'cfgmgr32.lib',
                      'powrprof.lib',
                      'setupapi.lib',
                      # net_common.gypi:
                      'crypt32.lib',
                      'dhcpcsvc.lib',
                      'rpcrt4.lib',
                      'secur32.lib',
                      'urlmon.lib',
                      'winhttp.lib',
                    ],
                    'DelayLoadDLLs': [
                      'wtsapi32.dll',
                      # content_common.gypi:
                      'd3d9.dll',
                      'd3d11.dll',
                      'dxva2.dll',
                      # media.gyp:
                      'mf.dll',
                      'mfplat.dll',
                      'mfreadwrite.dll',
                      # bluetooth.gyp:
                      'BluetoothApis.dll',
                      'Bthprops.cpl',
                      'setupapi.dll',
                      # base.gyp:
                      'cfgmgr32.dll',
                      'powrprof.dll',
                      'setupapi.dll',
                      # net_common.gypi:
                      'crypt32.dll',
                      'dhcpcsvc.dll',
                      'rpcrt4.dll',
                      'secur32.dll',
                      'urlmon.dll',
                      'winhttp.dll',
                      # windows runtime
                      'API-MS-WIN-CORE-WINRT-L1-1-0.DLL',
                      'API-MS-WIN-CORE-WINRT-STRING-L1-1-0.DLL',
                    ],
                  },
                },
              },
            }],  # libchromiumcontent_component
          ],
        }],  # OS=="win"
      ],
    },
  ],
}
<commit_msg>Always link dom_keycode_converter<commit_after>{
  'variables': {
    # The libraries brightray will be compiled to.
    'linux_system_libraries': 'gtk+-2.0 dbus-1 x11 xi xcursor xdamage xrandr xcomposite xext xfixes xrender xtst xscrnsaver gconf-2.0 gmodule-2.0 nss'
  },
  'includes': [
    'filenames.gypi',
  ],
  'targets': [
    {
      'target_name': 'brightray',
      'type': 'static_library',
      'include_dirs': [
        '.',
        '<(libchromiumcontent_src_dir)',
        '<(libchromiumcontent_src_dir)/skia/config',
        '<(libchromiumcontent_src_dir)/third_party/skia/include/core',
        '<(libchromiumcontent_src_dir)/third_party/mojo/src',
        '<(libchromiumcontent_src_dir)/third_party/WebKit',
        '<(libchromiumcontent_dir)/gen',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          '.',
          '..',
          '<(libchromiumcontent_src_dir)',
          '<(libchromiumcontent_src_dir)/gpu',
          '<(libchromiumcontent_src_dir)/skia/config',
          '<(libchromiumcontent_src_dir)/third_party/skia/include/core',
          '<(libchromiumcontent_src_dir)/third_party/skia/include/config',
          '<(libchromiumcontent_src_dir)/third_party/icu/source/common',
          '<(libchromiumcontent_src_dir)/third_party/mojo/src',
          '<(libchromiumcontent_src_dir)/third_party/khronos',
          '<(libchromiumcontent_src_dir)/third_party/WebKit',
          '<(libchromiumcontent_dir)/gen',
        ],
      },
      'sources': [ '<@(brightray_sources)' ],
      'conditions': [
        # Link with libraries of libchromiumcontent.
        ['OS=="linux" and libchromiumcontent_component==0', {
          # On Linux we have to use "--whole-archive" to force executable
          # to include all symbols, otherwise we will have plenty of
          # unresolved symbols errors.
          'direct_dependent_settings': {
            'ldflags': [
              '-Wl,--whole-archive',
              '<@(libchromiumcontent_libraries)',
              '-Wl,--no-whole-archive',
            ],
          }
        }, {  # (Release build on Linux)
          'link_settings': {
            'libraries': [ '<@(libchromiumcontent_libraries)' ]
          },
        }],  # (Normal builds)
        # Linux specific link settings.
        ['OS=="linux"', {
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other <(linux_system_libraries))',
            ],
            'libraries': [
              '-lpthread',
              '<!@(<(pkg-config) --libs-only-l <(linux_system_libraries))',
            ],
          },
          'cflags': [
            '<!@(<(pkg-config) --cflags <(linux_system_libraries))',
            # Needed by using libgtk2ui:
            '-Wno-deprecated-register',
            '-Wno-sentinel',
          ],
          'cflags_cc': [
            '-Wno-reserved-user-defined-literal',
          ],
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags <(linux_system_libraries))',
              '-Wno-deprecated-register',
              '-Wno-sentinel',
            ],
          },
          'conditions': [
            ['libchromiumcontent_component', {
              'link_settings': {
                'libraries': [
                  # Following libraries are always linked statically.
                  '<(libchromiumcontent_dir)/libgtk2ui.a',
                  '<(libchromiumcontent_dir)/libdevtools_discovery.a',
                  '<(libchromiumcontent_dir)/libdevtools_http_handler.a',
                  '<(libchromiumcontent_dir)/libhttp_server.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture_differ_sse2.a',
                  '<(libchromiumcontent_dir)/libdom_keycode_converter.a',
                  '<(libchromiumcontent_dir)/libsystem_wrappers.a',
                  '<(libchromiumcontent_dir)/librtc_base.a',
                  '<(libchromiumcontent_dir)/librtc_base_approved.a',
                  '<(libchromiumcontent_dir)/libwebrtc_common.a',
                  '<(libchromiumcontent_dir)/libyuv.a',
                  '<(libchromiumcontent_dir)/libcdm_renderer.a',
                  '<(libchromiumcontent_dir)/libsecurity_state.a',
                ],
              },
            }, {
              'link_settings': {
                'libraries': [
                  # Link with ffmpeg.
                  '<(libchromiumcontent_dir)/libffmpeg.so',
                  # Following libraries are required by libchromiumcontent:
                  '-lasound',
                  '-lcap',
                  '-lcups',
                  '-lrt',
                  '-ldl',
                  '-lresolv',
                  '-lfontconfig',
                  '-lfreetype',
                  '-lexpat',
                ],
              },
            }],
            ['target_arch=="arm"', {
              'link_settings': {
                'libraries!': [
                  '<(libchromiumcontent_dir)/libdesktop_capture_differ_sse2.a',
                ],
              },
            }],
          ],
        }],  # OS=="linux"
        ['OS=="mac"', {
          'link_settings': {
            'libraries': [
              '$(SDKROOT)/System/Library/Frameworks/AppKit.framework',
              # Required by webrtc:
              '$(SDKROOT)/System/Library/Frameworks/OpenGL.framework',
              '$(SDKROOT)/System/Library/Frameworks/IOKit.framework',
            ],
          },
          'conditions':  [
            ['libchromiumcontent_component', {
              'link_settings': {
                'libraries': [
                  # Following libraries are always linked statically.
                  '<(libchromiumcontent_dir)/libdevtools_discovery.a',
                  '<(libchromiumcontent_dir)/libdevtools_http_handler.a',
                  '<(libchromiumcontent_dir)/libhttp_server.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture.a',
                  '<(libchromiumcontent_dir)/libdesktop_capture_differ_sse2.a',
                  '<(libchromiumcontent_dir)/libdom_keycode_converter.a',
                  '<(libchromiumcontent_dir)/librtc_base.a',
                  '<(libchromiumcontent_dir)/librtc_base_approved.a',
                  '<(libchromiumcontent_dir)/libsystem_wrappers.a',
                  '<(libchromiumcontent_dir)/libwebrtc_common.a',
                  '<(libchromiumcontent_dir)/libyuv.a',
                  '<(libchromiumcontent_dir)/libcdm_renderer.a',
                  '<(libchromiumcontent_dir)/libsecurity_state.a',
                ],
              },
            }, {
              'link_settings': {
                'libraries': [
                  # Link with ffmpeg.
                  '<(libchromiumcontent_dir)/libffmpeg.dylib',
                  # Link with system frameworks.
                  # ui_base.gypi:
                  '$(SDKROOT)/System/Library/Frameworks/Accelerate.framework',
                  # net.gypi:
                  '$(SDKROOT)/System/Library/Frameworks/Foundation.framework',
                  '$(SDKROOT)/System/Library/Frameworks/Security.framework',
                  '$(SDKROOT)/System/Library/Frameworks/SystemConfiguration.framework',
                  '-lresolv',
                  # media.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/AudioToolbox.framework',
                  '$(SDKROOT)/System/Library/Frameworks/AudioUnit.framework',
                  '$(SDKROOT)/System/Library/Frameworks/AVFoundation.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreAudio.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreMedia.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreMIDI.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreVideo.framework',
                  # surface.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/IOSurface.framework',
                  # content_common.gypi:
                  '$(SDKROOT)/System/Library/Frameworks/QuartzCore.framework',
                  # base.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/ApplicationServices.framework',
                  '$(SDKROOT)/System/Library/Frameworks/Carbon.framework',
                  '$(SDKROOT)/System/Library/Frameworks/CoreFoundation.framework',
                  # content_browser.gypi:
                  '-lbsm',
                  # content_common.gypi:
                  '-lsandbox',
                  # bluetooth.gyp:
                  '$(SDKROOT)/System/Library/Frameworks/IOBluetooth.framework',
                ],
              },
            }],
          ]
        }],  # OS=="mac"
        ['OS=="win"', {
          'conditions': [
            ['libchromiumcontent_component', {
              'link_settings': {
                'libraries': [
                  # Needed by desktop_capture.lib:
                  '-ld3d11.lib',
                  # Following libs are always linked statically.
                  '<(libchromiumcontent_dir)/base_static.lib',
                  '<(libchromiumcontent_dir)/sandbox.lib',
                  '<(libchromiumcontent_dir)/sandbox_helper_win.lib',
                  '<(libchromiumcontent_dir)/devtools_discovery.lib',
                  '<(libchromiumcontent_dir)/devtools_http_handler.lib',
                  '<(libchromiumcontent_dir)/http_server.lib',
                  '<(libchromiumcontent_dir)/desktop_capture.lib',
                  '<(libchromiumcontent_dir)/desktop_capture_differ_sse2.lib',
                  '<(libchromiumcontent_dir)/dom_keycode_converter.lib',
                  '<(libchromiumcontent_dir)/rtc_base.lib',
                  '<(libchromiumcontent_dir)/rtc_base_approved.lib',
                  '<(libchromiumcontent_dir)/system_wrappers.lib',
                  '<(libchromiumcontent_dir)/webrtc_common.lib',
                  '<(libchromiumcontent_dir)/libyuv.lib',
                  '<(libchromiumcontent_dir)/cdm_renderer.lib',
                  '<(libchromiumcontent_dir)/security_state.lib',
                  # Friends of pdf.lib:
                  '<(libchromiumcontent_dir)/pdf.lib',
                  '<(libchromiumcontent_dir)/ppapi_cpp_objects.lib',
                  '<(libchromiumcontent_dir)/ppapi_internal_module.lib',
                  '<(libchromiumcontent_dir)/libjpeg.lib',
                  '<(libchromiumcontent_dir)/pdfium.lib',
                  '<(libchromiumcontent_dir)/fdrm.lib',
                  '<(libchromiumcontent_dir)/formfiller.lib',
                  '<(libchromiumcontent_dir)/fpdfapi.lib',
                  '<(libchromiumcontent_dir)/fpdfdoc.lib',
                  '<(libchromiumcontent_dir)/fpdftext.lib',
                  '<(libchromiumcontent_dir)/fpdftext.lib',
                  '<(libchromiumcontent_dir)/fxcodec.lib',
                  '<(libchromiumcontent_dir)/fxcrt.lib',
                  '<(libchromiumcontent_dir)/fxedit.lib',
                  '<(libchromiumcontent_dir)/fxge.lib',
                  '<(libchromiumcontent_dir)/javascript.lib',
                  '<(libchromiumcontent_dir)/pdfwindow.lib',
                  '<(libchromiumcontent_dir)/bigint.lib',
                  '<(libchromiumcontent_dir)/fx_agg.lib',
                  '<(libchromiumcontent_dir)/fx_freetype.lib',
                  '<(libchromiumcontent_dir)/fx_lcms2.lib',
                  '<(libchromiumcontent_dir)/fx_libopenjpeg.lib',
                  '<(libchromiumcontent_dir)/fx_zlib.lib',
                ],
              },
            }, {
              # Link with system libraries.
              'link_settings': {
                'libraries': [
                  # Link with ffmpeg.
                  '<(libchromiumcontent_dir)/ffmpeg.dll',
                  # content_browser.gypi:
                  '-lsensorsapi.lib',
                  '-lportabledeviceguids.lib',
                  # content_common.gypi:
                  '-ld3d9.lib',
                  '-ld3d11.lib',
                  '-ldxva2.lib',
                  '-lstrmiids.lib',
                  '-lmf.lib',
                  '-lmfplat.lib',
                  '-lmfuuid.lib',
                  # media.gyp:
                  '-ldxguid.lib',
                  '-lmfreadwrite.lib',
                  '-lmfuuid.lib',
                ],
                'msvs_settings': {
                  'VCLinkerTool': {
                    'AdditionalDependencies': [
                      'advapi32.lib',
                      'dbghelp.lib',
                      'delayimp.lib',
                      'dwmapi.lib',
                      'gdi32.lib',
                      'netapi32.lib',
                      'oleacc.lib',
                      'user32.lib',
                      'usp10.lib',
                      'version.lib',
                      'winspool.lib',
                      'wtsapi32.lib',
                      # bluetooth.gyp:
                      'Bthprops.lib',
                      'BluetoothApis.lib',
                      # base.gyp:
                      'cfgmgr32.lib',
                      'powrprof.lib',
                      'setupapi.lib',
                      # net_common.gypi:
                      'crypt32.lib',
                      'dhcpcsvc.lib',
                      'rpcrt4.lib',
                      'secur32.lib',
                      'urlmon.lib',
                      'winhttp.lib',
                    ],
                    'DelayLoadDLLs': [
                      'wtsapi32.dll',
                      # content_common.gypi:
                      'd3d9.dll',
                      'd3d11.dll',
                      'dxva2.dll',
                      # media.gyp:
                      'mf.dll',
                      'mfplat.dll',
                      'mfreadwrite.dll',
                      # bluetooth.gyp:
                      'BluetoothApis.dll',
                      'Bthprops.cpl',
                      'setupapi.dll',
                      # base.gyp:
                      'cfgmgr32.dll',
                      'powrprof.dll',
                      'setupapi.dll',
                      # net_common.gypi:
                      'crypt32.dll',
                      'dhcpcsvc.dll',
                      'rpcrt4.dll',
                      'secur32.dll',
                      'urlmon.dll',
                      'winhttp.dll',
                      # windows runtime
                      'API-MS-WIN-CORE-WINRT-L1-1-0.DLL',
                      'API-MS-WIN-CORE-WINRT-STRING-L1-1-0.DLL',
                    ],
                  },
                },
              },
            }],  # libchromiumcontent_component
          ],
        }],  # OS=="win"
      ],
    },
  ],
}
<|endoftext|>


<commit_before>"""Utility functions."""

import contextlib
import datetime
import glob
import os
import random
import shutil
import string
import subprocess
import sys
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

from . import config_dict_template

ENV = {}
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)

class FatalError(Exception):

    """A simple exception."""

    pass

def user_input(message):
    """Ask something to the user."""
    try:
        from builtins import input
    except ImportError:
        answer = raw_input(message)
    else:
        answer = input(message)
    return answer

def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs):
    """Execute a shell command.
    Run a command using the current user. Set :keyword:`sudo_user` if
    you need different privileges.
    :param str cmd: the command to execute
    :param str sudo_user: a valid system username
    :param str pinput: data to send to process's stdin
    :rtype: tuple
    :return: return code, command output
    """
    sudo_user = ENV.get("sudo_user", sudo_user)
    if sudo_user is not None:
        cmd = "sudo {}-u {} {}".format("-i " if login else "", sudo_user, cmd)
    if "shell" not in kwargs:
        kwargs["shell"] = True
    if pinput is not None:
        kwargs["stdin"] = subprocess.PIPE
    capture_output = False
    if "capture_output" in kwargs:
        capture_output = kwargs.pop("capture_output")
    elif not ENV.get("debug"):
        capture_output = True
    if capture_output:
        kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output = None
    process = subprocess.Popen(cmd, **kwargs)
    if pinput or capture_output:
        c_args = [pinput] if pinput is not None else []
        output = process.communicate(*c_args)[0]
    else:
        process.wait()
    return process.returncode, output

def dist_info():
    """Try to return information about the system we're running on."""
    path = "/etc/os-release"
    if os.path.exists(path):
        info = {}
        with open(path) as fp:
            while True:
                l = fp.readline()
                if not l:
                    break
                key, value = l.split("=")
                value = value.rstrip('"\n')
                value = value.strip('"')
                info[key] = value
        return info["NAME"], info["VERSION_ID"]
    printcolor(
        "Failed to retrieve information about your system, aborting.",
        RED)
    sys.exit(1)

def dist_name():
    """Try to guess the distribution name."""
    return dist_info()[0].lower()

def mkdir(path, mode, uid, gid):
    """Create a directory."""
    if not os.path.exists(path):
        os.mkdir(path, mode)
    else:
        os.chmod(path, mode)
    os.chown(path, uid, gid)

def make_password(length=16):
    """Create a random password."""
    return "".join(
        random.SystemRandom().choice(
            string.ascii_letters + string.digits) for _ in range(length))

@contextlib.contextmanager
def settings(**kwargs):
    """Context manager to declare temporary settings."""
    for key, value in kwargs.items():
        ENV[key] = value
    yield
    for key in kwargs.keys():
        del ENV[key]

class ConfigFileTemplate(string.Template):

    """Custom class for configuration files."""

    delimiter = "%"

def backup_file(fname):
    """Create a backup of a given file."""
    for f in glob.glob("{}.old.*".format(fname)):
        os.unlink(f)
    bak_name = "{}.old.{}".format(
        fname, datetime.datetime.now().isoformat())
    shutil.copy(fname, bak_name)

def copy_file(src, dest):
    """Copy a file to a destination and make a backup before."""
    if os.path.isdir(dest):
        dest = os.path.join(dest, os.path.basename(src))
    if os.path.isfile(dest):
        backup_file(dest)
    shutil.copy(src, dest)

def copy_from_template(template, dest, context):
    """Create and copy a configuration file from a template."""
    now = datetime.datetime.now().isoformat()
    with open(template) as fp:
        buf = fp.read()
    if os.path.isfile(dest):
        backup_file(dest)
    with open(dest, "w") as fp:
        fp.write(
            "# This file was automatically installed on {}\n"
            .format(now))
        fp.write(ConfigFileTemplate(buf).substitute(context))

def check_config_file(dest, interactive=False, upgrade=False):
    """Create a new installer config file if needed."""
    if os.path.exists(dest):
        return
    if upgrade:
        printcolor(
            "You cannot upgrade an existing installation without a "
            "configuration file.", RED)
        sys.exit(1)
    printcolor(
        "Configuration file {} not found, creating new one."
        .format(dest), YELLOW)
    gen_config(dest, interactive)

def has_colours(stream):
    """Check if terminal supports colors."""
    if not hasattr(stream, "isatty"):
        return False
    if not stream.isatty():
        return False  # auto color only on TTYs
    try:
        import curses
        curses.setupterm()
        return curses.tigetnum("colors") > 2
    except:
        # guess false in case of error
        return False

has_colours = has_colours(sys.stdout)

def printcolor(message, color):
    """Print a message using a green color."""
    if has_colours:
        message = "\x1b[1;{}m{}\x1b[0m".format(30 + color, message)
    print(message)

def convert_version_to_int(version):
    """Convert a version string to an integer."""
    number_bits = (8, 8, 16)

    numbers = [int(number_string) for number_string in version.split(".")]
    if len(numbers) > len(number_bits):
        raise NotImplementedError(
            "Versions with more than {0} decimal places are not supported"
            .format(len(number_bits) - 1)
        )
    # add 0s for missing numbers
    numbers.extend([0] * (len(number_bits) - len(numbers)))
    # convert to single int and return
    number = 0
    total_bits = 0
    for num, bits in reversed(list(zip(numbers, number_bits))):
        max_num = (bits + 1) - 1
        if num >= 1 << max_num:
            raise ValueError(
                "Number {0} cannot be stored with only {1} bits. Max is {2}"
                .format(num, bits, max_num)
            )
        number += num << total_bits
        total_bits += bits
    return number

def random_key(l=16):
    """Generate a random key.

    :param integer l: the key's length
    :return: a string
    """
    punctuation = """!#$%&()*+,-./:;<=>?@[]^_`{|}~"""
    population = string.digits + string.ascii_letters + punctuation
    while True:
        key = "".join(random.sample(population * l, l))
        if len(key) == l:
            return key

def validate(value, config_entry):
    if value is None:
        return False
    if "values" not in config_entry and "validators" not in config_entry:
        return True
    if "values" in config_entry:
        try:
            value = int(value)
        except ValueError:
            return False
        return value >= 0 and value < len(config_entry["values"])
    if "validators" in config_entry:
        for validator in config_entry["validators"]:
            valide, message = validator(value)
            if not valide:
                printcolor(message, MAGENTA)
                return False
        return True

def get_entry_value(entry, interactive):
    if callable(entry["default"]):
        default_value = entry["default"]()
    else:
        default_value = entry["default"]
    user_value = None
    if entry.get("customizable") and interactive:
        while (user_value != '' and not validate(user_value, entry)):
            question = entry.get("question")
            if entry.get("values"):
                question += " from the list"
                values = entry.get("values")
                for index, value in enumerate(values):
                    question += "\n{}   {}".format(index, value)
            print(question)
            print("default is <{}>".format(default_value))
            user_value = user_input("-> ")

        if entry.get("values") and user_value != "":
            user_value = values[int(user_value)]
    return user_value if user_value else default_value

def gen_config(dest, interactive=False):
    """Create config file from dict template"""
    tpl_dict = config_dict_template.ConfigDictTemplate
    config = configparser.ConfigParser()
    # only ask about options we need, else still generate default
    for section in tpl_dict:
        if "if" in section:
            config_key, value = section.get("if").split("=")
            section_name, option = config_key.split(".")
            interactive_section = (
                config.get(section_name, option) == value and interactive)
        else:
            interactive_section = interactive
        config.add_section(section["name"])
        for config_entry in section["values"]:
            value = get_entry_value(config_entry, interactive_section)
            config.set(section["name"], config_entry["option"], value)

    with open(dest, "w") as configfile:
        config.write(configfile)
<commit_msg>Fixed /etc/os-release parsing.<commit_after>"""Utility functions."""

import contextlib
import datetime
import glob
import os
import random
import shutil
import string
import subprocess
import sys
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

from . import config_dict_template

ENV = {}
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)

class FatalError(Exception):

    """A simple exception."""

    pass

def user_input(message):
    """Ask something to the user."""
    try:
        from builtins import input
    except ImportError:
        answer = raw_input(message)
    else:
        answer = input(message)
    return answer

def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs):
    """Execute a shell command.
    Run a command using the current user. Set :keyword:`sudo_user` if
    you need different privileges.
    :param str cmd: the command to execute
    :param str sudo_user: a valid system username
    :param str pinput: data to send to process's stdin
    :rtype: tuple
    :return: return code, command output
    """
    sudo_user = ENV.get("sudo_user", sudo_user)
    if sudo_user is not None:
        cmd = "sudo {}-u {} {}".format("-i " if login else "", sudo_user, cmd)
    if "shell" not in kwargs:
        kwargs["shell"] = True
    if pinput is not None:
        kwargs["stdin"] = subprocess.PIPE
    capture_output = False
    if "capture_output" in kwargs:
        capture_output = kwargs.pop("capture_output")
    elif not ENV.get("debug"):
        capture_output = True
    if capture_output:
        kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output = None
    process = subprocess.Popen(cmd, **kwargs)
    if pinput or capture_output:
        c_args = [pinput] if pinput is not None else []
        output = process.communicate(*c_args)[0]
    else:
        process.wait()
    return process.returncode, output

def dist_info():
    """Try to return information about the system we're running on."""
    path = "/etc/os-release"
    if os.path.exists(path):
        info = {}
        with open(path) as fp:
            for line in fp.readlines():
                if line == '\n':
                    continue
                key, value = line.split("=")
                value = value.rstrip('"\n')
                value = value.strip('"')
                info[key] = value
        return info["NAME"], info["VERSION_ID"]
    printcolor(
        "Failed to retrieve information about your system, aborting.",
        RED)
    sys.exit(1)

def dist_name():
    """Try to guess the distribution name."""
    return dist_info()[0].lower()

def mkdir(path, mode, uid, gid):
    """Create a directory."""
    if not os.path.exists(path):
        os.mkdir(path, mode)
    else:
        os.chmod(path, mode)
    os.chown(path, uid, gid)

def make_password(length=16):
    """Create a random password."""
    return "".join(
        random.SystemRandom().choice(
            string.ascii_letters + string.digits) for _ in range(length))

@contextlib.contextmanager
def settings(**kwargs):
    """Context manager to declare temporary settings."""
    for key, value in kwargs.items():
        ENV[key] = value
    yield
    for key in kwargs.keys():
        del ENV[key]

class ConfigFileTemplate(string.Template):

    """Custom class for configuration files."""

    delimiter = "%"

def backup_file(fname):
    """Create a backup of a given file."""
    for f in glob.glob("{}.old.*".format(fname)):
        os.unlink(f)
    bak_name = "{}.old.{}".format(
        fname, datetime.datetime.now().isoformat())
    shutil.copy(fname, bak_name)

def copy_file(src, dest):
    """Copy a file to a destination and make a backup before."""
    if os.path.isdir(dest):
        dest = os.path.join(dest, os.path.basename(src))
    if os.path.isfile(dest):
        backup_file(dest)
    shutil.copy(src, dest)

def copy_from_template(template, dest, context):
    """Create and copy a configuration file from a template."""
    now = datetime.datetime.now().isoformat()
    with open(template) as fp:
        buf = fp.read()
    if os.path.isfile(dest):
        backup_file(dest)
    with open(dest, "w") as fp:
        fp.write(
            "# This file was automatically installed on {}\n"
            .format(now))
        fp.write(ConfigFileTemplate(buf).substitute(context))

def check_config_file(dest, interactive=False, upgrade=False):
    """Create a new installer config file if needed."""
    if os.path.exists(dest):
        return
    if upgrade:
        printcolor(
            "You cannot upgrade an existing installation without a "
            "configuration file.", RED)
        sys.exit(1)
    printcolor(
        "Configuration file {} not found, creating new one."
        .format(dest), YELLOW)
    gen_config(dest, interactive)

def has_colours(stream):
    """Check if terminal supports colors."""
    if not hasattr(stream, "isatty"):
        return False
    if not stream.isatty():
        return False  # auto color only on TTYs
    try:
        import curses
        curses.setupterm()
        return curses.tigetnum("colors") > 2
    except:
        # guess false in case of error
        return False

has_colours = has_colours(sys.stdout)

def printcolor(message, color):
    """Print a message using a green color."""
    if has_colours:
        message = "\x1b[1;{}m{}\x1b[0m".format(30 + color, message)
    print(message)

def convert_version_to_int(version):
    """Convert a version string to an integer."""
    number_bits = (8, 8, 16)

    numbers = [int(number_string) for number_string in version.split(".")]
    if len(numbers) > len(number_bits):
        raise NotImplementedError(
            "Versions with more than {0} decimal places are not supported"
            .format(len(number_bits) - 1)
        )
    # add 0s for missing numbers
    numbers.extend([0] * (len(number_bits) - len(numbers)))
    # convert to single int and return
    number = 0
    total_bits = 0
    for num, bits in reversed(list(zip(numbers, number_bits))):
        max_num = (bits + 1) - 1
        if num >= 1 << max_num:
            raise ValueError(
                "Number {0} cannot be stored with only {1} bits. Max is {2}"
                .format(num, bits, max_num)
            )
        number += num << total_bits
        total_bits += bits
    return number

def random_key(l=16):
    """Generate a random key.

    :param integer l: the key's length
    :return: a string
    """
    punctuation = """!#$%&()*+,-./:;<=>?@[]^_`{|}~"""
    population = string.digits + string.ascii_letters + punctuation
    while True:
        key = "".join(random.sample(population * l, l))
        if len(key) == l:
            return key

def validate(value, config_entry):
    if value is None:
        return False
    if "values" not in config_entry and "validators" not in config_entry:
        return True
    if "values" in config_entry:
        try:
            value = int(value)
        except ValueError:
            return False
        return value >= 0 and value < len(config_entry["values"])
    if "validators" in config_entry:
        for validator in config_entry["validators"]:
            valide, message = validator(value)
            if not valide:
                printcolor(message, MAGENTA)
                return False
        return True

def get_entry_value(entry, interactive):
    if callable(entry["default"]):
        default_value = entry["default"]()
    else:
        default_value = entry["default"]
    user_value = None
    if entry.get("customizable") and interactive:
        while (user_value != '' and not validate(user_value, entry)):
            question = entry.get("question")
            if entry.get("values"):
                question += " from the list"
                values = entry.get("values")
                for index, value in enumerate(values):
                    question += "\n{}   {}".format(index, value)
            print(question)
            print("default is <{}>".format(default_value))
            user_value = user_input("-> ")

        if entry.get("values") and user_value != "":
            user_value = values[int(user_value)]
    return user_value if user_value else default_value

def gen_config(dest, interactive=False):
    """Create config file from dict template"""
    tpl_dict = config_dict_template.ConfigDictTemplate
    config = configparser.ConfigParser()
    # only ask about options we need, else still generate default
    for section in tpl_dict:
        if "if" in section:
            config_key, value = section.get("if").split("=")
            section_name, option = config_key.split(".")
            interactive_section = (
                config.get(section_name, option) == value and interactive)
        else:
            interactive_section = interactive
        config.add_section(section["name"])
        for config_entry in section["values"]:
            value = get_entry_value(config_entry, interactive_section)
            config.set(section["name"], config_entry["option"], value)

    with open(dest, "w") as configfile:
        config.write(configfile)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Exercício de avaliação de expressão aritmética.
# Só podem ser usadas as estruturas Pilha e Fila implementadas em aulas anteriores.
# Deve ter análise de tempo e espaço para função avaliação

from aula5.fila import Fila

class ErroLexico(Exception):
    pass

class ErroSintatico(Exception):
    pass

def analise_lexica(expressao):
    """
    Executa análise lexica transformando a expressao em fila de objetos:
    Transforma inteiros em ints
    Flutuantes em floats
    e verificar se demais caracteres são validos: +-*/(){}[]
    :param expressao: string com expressao a ser analisada
    :return: fila com tokens
    """
    fila = Fila()

    #numero e tolkens
    nt = R"0123456789.-+*/{}[]()"

    print (nt)

    if expressao:
        for i in expressao:
            if i in nt:
                fila.enfileirar()

    return fila

def analise_sintatica(fila):
    """
    Função que realiza analise sintática de tokens produzidos por analise léxica.
    Executa validações sintáticas e se não houver erro retorn fila_sintatica para avaliacao

    :param fila: fila proveniente de análise lexica
    :return: fila_sintatica com elementos tokens de numeros
    """
    pass

def avaliar(expressao):
    """
    Função que avalia expressão aritmetica retornando se valor se não houver nenhum erro
    :param expressao: string com expressão aritmética
    :return: valor númerico com resultado
    """
    pass

import unittest

class AnaliseLexicaTestes(unittest.TestCase):
    def test_expressao_vazia(self):
        fila = analise_lexica('')
        self.assertTrue(fila.vazia())

    def test_caracter_estranho(self):
        self.assertRaises(ErroLexico, analise_lexica, 'a')
        self.assertRaises(ErroLexico, analise_lexica, 'ab')

    def test_inteiro_com_um_algarismo(self):
        fila = analise_lexica('1')
        self.assertEqual('1', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_inteiro_com_vários_algarismos(self):
        fila = analise_lexica('1234567890')
        self.assertEqual('1234567890', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_float(self):
        fila = analise_lexica('1234567890.34')
        self.assertEqual('1234567890', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('34', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_parenteses(self):
        fila = analise_lexica('(1)')
        self.assertEqual('(', fila.desenfileirar())
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual(')', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_chaves(self):
        fila = analise_lexica('{(1)}')
        self.assertEqual('{', fila.desenfileirar())
        self.assertEqual('(', fila.desenfileirar())
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual(')', fila.desenfileirar())
        self.assertEqual('}', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_colchetes(self):
        fila = analise_lexica('[{(1.0)}]')
        self.assertEqual('[', fila.desenfileirar())
        self.assertEqual('{', fila.desenfileirar())
        self.assertEqual('(', fila.desenfileirar())
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertEqual(')', fila.desenfileirar())
        self.assertEqual('}', fila.desenfileirar())
        self.assertEqual(']', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_adicao(self):
        fila = analise_lexica('1+2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('+', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_subtracao(self):
        fila = analise_lexica('1-2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('-', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_multiplicacao(self):
        fila = analise_lexica('1*2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('*', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_divisao(self):
        fila = analise_lexica('1/2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('/', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_expresao_com_todos_simbolos(self):
        expressao = '1/{2.0+3*[7-(5-3)]}'
        fila = analise_lexica(expressao)
        self.assertListEqual(list(expressao), [e for e in fila])
        self.assertTrue(fila.vazia())

class AnaliseSintaticaTestes(unittest.TestCase):
    def test_fila_vazia(self):
        fila = Fila()
        self.assertRaises(ErroSintatico, analise_sintatica, fila)

    def test_int(self):
        fila = Fila()
        fila.enfileirar('1234567890')
        fila_sintatica = analise_sintatica(fila)
        self.assertEqual(1234567890, fila_sintatica.desenfileirar())
        self.assertTrue(fila_sintatica.vazia())

    def test_float(self):
        fila = Fila()
        fila.enfileirar('1234567890')
        fila.enfileirar('.')
        fila.enfileirar('4')
        fila_sintatica = analise_sintatica(fila)
        self.assertEqual(1234567890.4, fila_sintatica.desenfileirar())
        self.assertTrue(fila_sintatica.vazia())

    def test_expressao_com_todos_elementos(self):
        fila = analise_lexica('1000/{222.125+3*[7-(5-3)]}')
        fila_sintatica = analise_sintatica(fila)
        self.assertListEqual([1000, '/', '{', 222.125, '+', 3, '*', '[', 7, '-', '(', 5, '-', 3, ')', ']', '}'],
                             [e for e in fila_sintatica])

class AvaliacaoTestes(unittest.TestCase):
    def test_expressao_vazia(self):
        self.assertRaises(ErroSintatico, avaliar, '')

    def test_inteiro(self):
        self.assert_avaliacao('1')

    def test_float(self):
        self.assert_avaliacao('2.1')

    def test_soma(self):
        self.assert_avaliacao('2+1')

    def test_subtracao_e_parenteses(self):
        self.assert_avaliacao('(2-1)')

    def test_expressao_com_todos_elementos(self):
        self.assertEqual(1.0, avaliar('2.0/[4*3+1-{15-(1+3)}]'))

    def assert_avaliacao(self, expressao):
        self.assertEqual(eval(expressao), avaliar(expressao))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Analise léxica OK<commit_after># -*- coding: utf-8 -*-

# Exercício de avaliação de expressão aritmética.
# Só podem ser usadas as estruturas Pilha e Fila implementadas em aulas anteriores.
# Deve ter análise de tempo e espaço para função avaliação

from aula5.fila import Fila

class ErroLexico(Exception):
    pass

class ErroSintatico(Exception):
    pass

def analise_lexica(expressao):
    """
    Executa análise lexica transformando a expressao em fila de objetos:
    Transforma inteiros em ints
    Flutuantes em floats
    e verificar se demais caracteres são validos: +-*/(){}[]
    :param expressao: string com expressao a ser analisada
    :return: fila com tokens
    """
    fila = Fila()

    #numero e tolkens
    nt = R"0123456789.-+*/{}[]()"

    if expressao:

        valor = ''

        for i in expressao:
            if i in nt:
                if i in '.-+*/{}[]()':
                    if valor:
                        fila.enfileirar(valor)
                        valor = ''
                    fila.enfileirar(i)
                else:
                    valor = valor + i
            else:
                raise ErroLexico()

        if valor:
            fila.enfileirar(valor)

    return fila

def analise_sintatica(fila):
    """
    Função que realiza analise sintática de tokens produzidos por analise léxica.
    Executa validações sintáticas e se não houver erro retorn fila_sintatica para avaliacao

    :param fila: fila proveniente de análise lexica
    :return: fila_sintatica com elementos tokens de numeros
    """
    pass

def avaliar(expressao):
    """
    Função que avalia expressão aritmetica retornando se valor se não houver nenhum erro
    :param expressao: string com expressão aritmética
    :return: valor númerico com resultado
    """
    pass

import unittest

class AnaliseLexicaTestes(unittest.TestCase):
    def test_expressao_vazia(self):
        fila = analise_lexica('')
        self.assertTrue(fila.vazia())

    def test_caracter_estranho(self):
        self.assertRaises(ErroLexico, analise_lexica, 'a')
        self.assertRaises(ErroLexico, analise_lexica, 'ab')

    def test_inteiro_com_um_algarismo(self):
        fila = analise_lexica('1')
        self.assertEqual('1', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_inteiro_com_vários_algarismos(self):
        fila = analise_lexica('1234567890')
        self.assertEqual('1234567890', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_float(self):
        fila = analise_lexica('1234567890.34')
        self.assertEqual('1234567890', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('34', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_parenteses(self):
        fila = analise_lexica('(1)')
        self.assertEqual('(', fila.desenfileirar())
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual(')', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_chaves(self):
        fila = analise_lexica('{(1)}')
        self.assertEqual('{', fila.desenfileirar())
        self.assertEqual('(', fila.desenfileirar())
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual(')', fila.desenfileirar())
        self.assertEqual('}', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_colchetes(self):
        fila = analise_lexica('[{(1.0)}]')
        self.assertEqual('[', fila.desenfileirar())
        self.assertEqual('{', fila.desenfileirar())
        self.assertEqual('(', fila.desenfileirar())
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertEqual(')', fila.desenfileirar())
        self.assertEqual('}', fila.desenfileirar())
        self.assertEqual(']', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_adicao(self):
        fila = analise_lexica('1+2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('+', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_subtracao(self):
        fila = analise_lexica('1-2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('-', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_multiplicacao(self):
        fila = analise_lexica('1*2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('*', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_divisao(self):
        fila = analise_lexica('1/2.0')
        self.assertEqual('1', fila.desenfileirar())
        self.assertEqual('/', fila.desenfileirar())
        self.assertEqual('2', fila.desenfileirar())
        self.assertEqual('.', fila.desenfileirar())
        self.assertEqual('0', fila.desenfileirar())
        self.assertTrue(fila.vazia())

    def test_expresao_com_todos_simbolos(self):
        expressao = '1/{2.0+3*[7-(5-3)]}'
        fila = analise_lexica(expressao)
        self.assertListEqual(list(expressao), [e for e in fila])
        self.assertTrue(fila.vazia())

class AnaliseSintaticaTestes(unittest.TestCase):
    def test_fila_vazia(self):
        fila = Fila()
        self.assertRaises(ErroSintatico, analise_sintatica, fila)

    def test_int(self):
        fila = Fila()
        fila.enfileirar('1234567890')
        fila_sintatica = analise_sintatica(fila)
        self.assertEqual(1234567890, fila_sintatica.desenfileirar())
        self.assertTrue(fila_sintatica.vazia())

    def test_float(self):
        fila = Fila()
        fila.enfileirar('1234567890')
        fila.enfileirar('.')
        fila.enfileirar('4')
        fila_sintatica = analise_sintatica(fila)
        self.assertEqual(1234567890.4, fila_sintatica.desenfileirar())
        self.assertTrue(fila_sintatica.vazia())

    def test_expressao_com_todos_elementos(self):
        fila = analise_lexica('1000/{222.125+3*[7-(5-3)]}')
        fila_sintatica = analise_sintatica(fila)
        self.assertListEqual([1000, '/', '{', 222.125, '+', 3, '*', '[', 7, '-', '(', 5, '-', 3, ')', ']', '}'],
                             [e for e in fila_sintatica])

class AvaliacaoTestes(unittest.TestCase):
    def test_expressao_vazia(self):
        self.assertRaises(ErroSintatico, avaliar, '')

    def test_inteiro(self):
        self.assert_avaliacao('1')

    def test_float(self):
        self.assert_avaliacao('2.1')

    def test_soma(self):
        self.assert_avaliacao('2+1')

    def test_subtracao_e_parenteses(self):
        self.assert_avaliacao('(2-1)')

    def test_expressao_com_todos_elementos(self):
        self.assertEqual(1.0, avaliar('2.0/[4*3+1-{15-(1+3)}]'))

    def assert_avaliacao(self, expressao):
        self.assertEqual(eval(expressao), avaliar(expressao))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from django.core.management import call_command
from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = "Load initial data into the master db"

    def handle(self, *args, **options):
        call_command('loaddata',
                     'repository_group',
                     'repository',
                     'failure_classification',
                     'performance_framework',
                     'performance_bug_templates')
        call_command('update_runnable_jobs')
        call_command('load_preseed')
<commit_msg>Bug 1329685 - Stop calling update_runnable_jobs during load_initial_data<commit_after>from django.core.management import call_command
from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = "Load initial data into the master db"

    def handle(self, *args, **options):
        call_command('loaddata',
                     'repository_group',
                     'repository',
                     'failure_classification',
                     'performance_framework',
                     'performance_bug_templates')
        call_command('load_preseed')
<|endoftext|>