<commit_before>"""Configure search catalogs."""
from zope.interface import Interface
from pyramid.registry import Registry
from itertools import islice
from collections.abc import Iterable
from substanced import catalog
from substanced.interfaces import IIndexingActionProcessor
from substanced.catalog import CatalogsService
from hypatia.interfaces import IIndex
from hypatia.interfaces import IResultSet
from hypatia.util import ResultSet
from adhocracy_core.interfaces import IServicePool
from adhocracy_core.interfaces import FieldComparator
from adhocracy_core.interfaces import FieldSequenceComparator
from adhocracy_core.interfaces import KeywordComparator
from adhocracy_core.interfaces import KeywordSequenceComparator
from adhocracy_core.interfaces import SearchResult
from adhocracy_core.interfaces import SearchQuery
from adhocracy_core.interfaces import search_result
from adhocracy_core.interfaces import IResource
from adhocracy_core.resources.service import service_meta
from adhocracy_core.resources import add_resource_type_to_registry
from adhocracy_core.utils import normalize_to_tuple

class ICatalogsService(IServicePool):
    """The 'catalogs' ServicePool."""

class CatalogsServiceAdhocracy(CatalogsService):

    def reindex_all(self, resource: IResource):
        """Reindex `resource` with all indexes."""
        for value in self.values():
            value.reindex_resource(resource)

    def reindex_index(self, resource: IResource, index_name: str):
        """Reindex `resource` with index `index_name`.

        :raises KeyError: if `index_name`  index does not exists.
        """
        index = self.get_index(index_name)
        if index is None:
            msg = 'catalog index {0} does not exist.'.format(index_name)
            raise KeyError(msg)
        index.reindex_resource(resource)

    def search(self, query: SearchQuery) -> SearchResult:
        """Search indexes in catalogs `adhocracy` and `system`."""
        elements = self._search_elements(query)
        frequency_of = self._get_frequency_of(elements, query)
        group_by = self._get_group_by(elements, query)
        sorted_elements = self._sort_elements(elements, query)
        count = len(sorted_elements)
        elements_slice = self._get_slice(sorted_elements, query)
        resolved = self._resolve(elements_slice, query)
        result = search_result._replace(elements=resolved,
                                        count=count,
                                        group_by=group_by,
                                        frequency_of=frequency_of)
        return result

    def _search_elements(self, query) -> IResultSet:
        interfaces_index = self.get_index('interfaces')
        if interfaces_index is None:  # pragma: no branch
            return ResultSet(set(), 0, None)
        interfaces_value = self._get_query_value(query.interfaces)
        if not interfaces_value:
            interfaces_value = (IResource,)
        interfaces_comparator = self._get_query_comparator(query.interfaces)
        if interfaces_comparator is None:
            interfaces_value = normalize_to_tuple(interfaces_value)
            index_query = interfaces_index.all(interfaces_value)
        else:
            index_comparator = getattr(interfaces_index, interfaces_comparator)
            index_query = index_comparator(interfaces_value)
        if query.root is not None:
            depth = query.depth or None
            path_index = self.get_index('path')
            index_query &= path_index.eq(query.root,
                                         depth=depth,
                                         include_origin=False)
        if query.indexes:
            for index_name, value in query.indexes.items():
                index = self.get_index(index_name)
                comparator = self._get_query_comparator(value)
                if comparator is None:
                    index_comparator = index.eq
                else:
                    index_comparator = getattr(index, comparator)
                index_value = self._get_query_value(value)
                index_query &= index_comparator(index_value)
        if query.only_visible:
            visibility_index = self.get_index('private_visibility')
            index_query &= visibility_index.eq('visible')
        if query.allows:
            allowed_index = self.get_index('allowed')
            principals, permission = query.allows
            index_query &= allowed_index.allows(principals, permission)
        elements = index_query.execute(resolver=None)
        if query.references:
            index = self.get_index('reference')
            for reference in query.references:
                referencence_elements = index.search_with_order(reference)
                referencence_elements.resolver = elements.resolver
                elements = referencence_elements.intersect(elements)
        return elements

    def _get_frequency_of(self, elements: IResultSet,
                          query: SearchQuery) -> dict:
        frequency_of = {}
        if query.frequency_of:
            index = self.get_index(query.frequency_of)
            for value in index.unique_values():
                value_query = index.eq(value)
                value_elements = value_query.execute(resolver=None)
                intersect = elements.intersect(value_elements)
                count = len(intersect)
                if count == 0:
                    continue
                frequency_of[value] = count
        return frequency_of

    def _get_group_by(self, elements: IResultSet, query: SearchQuery) -> dict:
        group_by = {}
        if query.group_by:
            index = self.get_index(query.group_by)
            for value in index.unique_values():
                value_query = index.eq(value)
                value_elements = value_query.execute(resolver=None)
                intersect = elements.intersect(value_elements)
                if len(intersect) == 0:
                    continue
                group_by[value] = intersect
        sort_index = self.get_index(query.sort_by)
        if sort_index is not None:
            for key, intersect in group_by.items():
                intersect_sorted = intersect.sort(sort_index,
                                                  reverse=query.reverse,
                                                  limit=query.limit or None)
                group_by[key] = intersect_sorted
        if query.resolve:
            for key, intersect in group_by.items():
                intersect_resolved = [x for x in intersect]
                group_by[key] = intersect_resolved
        return group_by

    def _sort_elements(self, elements: IResultSet,
                       query: SearchQuery) -> IResultSet:
        sort_index = self.get_index(query.sort_by)
        if sort_index is not None:
            # TODO: We should assert the IIndexSort interface here, but
            # hypatia.field.FieldIndex is missing this interface.
            assert 'sort' in sort_index.__dir__()
            elements = elements.sort(sort_index,
                                     reverse=query.reverse,
                                     limit=query.limit or None)
        return elements

    def _get_slice(self, elements: IResultSet, query: IResultSet) -> Iterable:
        """Get slice defined by `query.limit` and `query.offset`.

        :returns: IResultSet if not `query.limit`, else [IResource]
        """
        elements_slice = elements
        if query.limit:
            docids_slice = islice(elements.all(resolve=None),
                                  query.offset,
                                  query.offset + query.limit)
            elements_slice = [elements.resolver(x) for x in docids_slice]
        return elements_slice

    def _resolve(self, elements: Iterable, query: SearchQuery) -> Iterable:
        if query.resolve:
            elements = [x for x in elements]
        return elements

    def get_index(self, name) -> IIndex:
        system = self.get('system', {})
        adhocracy = self.get('adhocracy', {})
        index = system.get(name, None) or adhocracy.get(name, None)
        return index

    def _get_query_value(self, query_parameter: tuple) -> object:
        if self._is_tuple_starting_with_comparator(query_parameter):
            return query_parameter[1]
        else:
            return query_parameter

    def _get_query_comparator(self, query_parameter: tuple) -> object:
        if self._is_tuple_starting_with_comparator(query_parameter):
            return query_parameter[0]
        else:
            return

    def _is_tuple_starting_with_comparator(self, parameter: tuple) -> bool:
        if not isinstance(parameter, tuple):
            return False
        elif len(parameter) != 2:
            return False
        elif parameter[0] in self._comparators:
            return True

    _comparators = set(FieldComparator.__members__) \
        .union(KeywordComparator.__members__) \
        .union(KeywordSequenceComparator.__members__) \
        .union(FieldSequenceComparator.__members__)

def add_catalogs_system_and_adhocracy(context: ICatalogsService,
                                      registry: Registry,
                                      options: dict):
    """Add catalogs 'system' and 'adhocracy'."""
    context.add_catalog('system')
    context.add_catalog('adhocracy')

catalogs_service_meta = service_meta._replace(
    iresource=ICatalogsService,
    content_name='catalogs',
    content_class=CatalogsServiceAdhocracy,
    after_creation=[add_catalogs_system_and_adhocracy]
)

def includeme(config):
    """Register catalog utilities."""
    config.include('adhocracy_core.events')
    config.add_view_predicate('catalogable', catalog._CatalogablePredicate)
    config.add_directive('add_catalog_factory', catalog.add_catalog_factory)
    config.add_directive('add_indexview',
                         catalog.add_indexview,
                         action_wrap=False)
    config.registry.registerAdapter(catalog.deferred.BasicActionProcessor,
                                    (Interface,),
                                    IIndexingActionProcessor)
    add_resource_type_to_registry(catalogs_service_meta, config)
    config.scan('substanced.catalog')
    config.scan('.index')
    config.include('.adhocracy')
    config.include('.subscriber')
<commit_msg>Refactor catalog search<commit_after>"""Configure search catalogs."""
from itertools import chain

from zope.interface import Interface
from pyramid.registry import Registry
from itertools import islice
from collections.abc import Iterable
from substanced import catalog
from substanced.interfaces import IIndexingActionProcessor
from substanced.catalog import CatalogsService
from hypatia.interfaces import IIndex
from hypatia.interfaces import IResultSet
from hypatia.util import ResultSet
from adhocracy_core.interfaces import IServicePool
from adhocracy_core.interfaces import FieldComparator
from adhocracy_core.interfaces import FieldSequenceComparator
from adhocracy_core.interfaces import KeywordComparator
from adhocracy_core.interfaces import KeywordSequenceComparator
from adhocracy_core.interfaces import SearchResult
from adhocracy_core.interfaces import SearchQuery
from adhocracy_core.interfaces import search_result
from adhocracy_core.interfaces import IResource
from adhocracy_core.resources.service import service_meta
from adhocracy_core.resources import add_resource_type_to_registry
from adhocracy_core.utils import normalize_to_tuple

class ICatalogsService(IServicePool):
    """The 'catalogs' ServicePool."""

class CatalogsServiceAdhocracy(CatalogsService):

    def reindex_all(self, resource: IResource):
        """Reindex `resource` with all indexes."""
        for value in self.values():
            value.reindex_resource(resource)

    def reindex_index(self, resource: IResource, index_name: str):
        """Reindex `resource` with index `index_name`.

        :raises KeyError: if `index_name`  index does not exists.
        """
        index = self.get_index(index_name)
        if index is None:
            msg = 'catalog index {0} does not exist.'.format(index_name)
            raise KeyError(msg)
        index.reindex_resource(resource)

    def search(self, query: SearchQuery) -> SearchResult:
        """Search indexes in catalogs `adhocracy` and `system`."""
        elements = self._search_elements(query)
        frequency_of = self._get_frequency_of(elements, query)
        group_by = self._get_group_by(elements, query)
        sorted_elements = self._sort_elements(elements, query)
        count = len(sorted_elements)
        elements_slice = self._get_slice(sorted_elements, query)
        resolved = self._resolve(elements_slice, query)
        result = search_result._replace(elements=resolved,
                                        count=count,
                                        group_by=group_by,
                                        frequency_of=frequency_of)
        return result

    def _get_interfaces_index_query(self, query):
        interfaces_value = self._get_query_value(query.interfaces)
        # if not interfaces_value and query.references:
        #     # do not search for all IResource and then intersect with
        #     # the references when searching for references with
        #     # non-specified interfaces
        #     return None
        if not interfaces_value:
            interfaces_value = (IResource,)
        interfaces_comparator = self._get_query_comparator(query.interfaces)
        interfaces_index = self.get_index('interfaces')
        if interfaces_comparator is None:
            interfaces_value = normalize_to_tuple(interfaces_value)
            index_query = interfaces_index.all(interfaces_value)
        else:
            index_comparator = getattr(interfaces_index, interfaces_comparator)
            index_query = index_comparator(interfaces_value)
        return index_query

    def _get_path_index_query(self, query):
        if query.root is None:
            return None
        depth = query.depth or None
        path_index = self.get_index('path')
        return path_index.eq(query.root,
                             depth=depth,
                             include_origin=False)

    def _get_indexes_index_query(self, query):
        if not query.indexes:
            return []
        indexes = []
        for index_name, value in query.indexes.items():
                index = self.get_index(index_name)
                comparator = self._get_query_comparator(value)
                if comparator is None:
                    index_comparator = index.eq
                else:
                    index_comparator = getattr(index, comparator)
                index_value = self._get_query_value(value)
                indexes.append(index_comparator(index_value))
        return indexes

    def _get_private_visibility_index_query(self, query):
        if not query.only_visible:
            return None
        visibility_index = self.get_index('private_visibility')
        return visibility_index.eq('visible')

    def _get_allowed_index_query(self, query):
        if not query.allows:
            return None
        allowed_index = self.get_index('allowed')
        principals, permission = query.allows
        return allowed_index.allows(principals, permission)

    def _combine_indexes(self, query, *list_of_indexes):
        maybes_indexes = chain.from_iterable(list_of_indexes)
        indexes = [idx for idx in maybes_indexes if idx is not None]
        return indexes

    def _execute_query(self, indexes):
        if len(indexes) > 0:
            index_query = indexes[0]
            for idx in indexes[1:]:
                index_query &= idx
            elements = index_query.execute(resolver=None)
            elements.all(resolve=True)
        else:
            elements = ResultSet(set(), 0, None)
        return elements

    def _search_references(self, query, resolver):
        if not query.references:
            res = ResultSet(set(), 0, None)
            res.resolver = resolver
            return res
        index = self.get_index('reference')
        accumulated_refs = ResultSet(set(), 0, None)
        accumulated_refs = index.search_with_order(query.references[0])
        accumulated_refs.resolver = resolver
        for reference in query.references[1:]:
             found = index.search_with_order(reference)
             found.resolver = resolver
             accumulated_refs = found.intersect(accumulated_refs)
        return accumulated_refs

    def _combine_results(self, query, elements, references):
        if not query.references:
            return elements
        elif query.interfaces and query.references:
            return elements.intersect(references)
        elif not query.interfaces and query.references:
            return references
        else:
            return elements

    @profile
    def _search_elements(self, query) -> IResultSet:
#        import pudb; pudb.set_trace() #  noqa
        interfaces_index = self.get_index('interfaces')
        if interfaces_index is None:  # pragma: no branch
            return ResultSet(set(), 0, None)
        indexes = self._combine_indexes(
            query,
            [self._get_interfaces_index_query(query)],
            [self._get_path_index_query(query)],
            self._get_indexes_index_query(query),
            [self._get_private_visibility_index_query(query)],
            [self._get_allowed_index_query(query)],)
        # TODO: build_query
        elements = self._execute_query(indexes)
        references = self._search_references(query, elements.resolver)
        result = self._combine_results(query, elements, references)
#        profile.print_stats()
# TODO: see why yield references is called so many times

        return result

    def _get_frequency_of(self, elements: IResultSet,
                          query: SearchQuery) -> dict:
        frequency_of = {}
        if query.frequency_of:
            index = self.get_index(query.frequency_of)
            for value in index.unique_values():
                value_query = index.eq(value)
                value_elements = value_query.execute(resolver=None)
                intersect = elements.intersect(value_elements)
                count = len(intersect)
                if count == 0:
                    continue
                frequency_of[value] = count
        return frequency_of

    def _get_group_by(self, elements: IResultSet, query: SearchQuery) -> dict:
        group_by = {}
        if query.group_by:
            index = self.get_index(query.group_by)
            for value in index.unique_values():
                value_query = index.eq(value)
                value_elements = value_query.execute(resolver=None)
                intersect = elements.intersect(value_elements)
                if len(intersect) == 0:
                    continue
                group_by[value] = intersect
        sort_index = self.get_index(query.sort_by)
        if sort_index is not None:
            for key, intersect in group_by.items():
                intersect_sorted = intersect.sort(sort_index,
                                                  reverse=query.reverse,
                                                  limit=query.limit or None)
                group_by[key] = intersect_sorted
        if query.resolve:
            for key, intersect in group_by.items():
                intersect_resolved = [x for x in intersect]
                group_by[key] = intersect_resolved
        return group_by

    def _sort_elements(self, elements: IResultSet,
                       query: SearchQuery) -> IResultSet:
        sort_index = self.get_index(query.sort_by)
        if sort_index is not None:
            # TODO: We should assert the IIndexSort interface here, but
            # hypatia.field.FieldIndex is missing this interface.
            assert 'sort' in sort_index.__dir__()
            elements = elements.sort(sort_index,
                                     reverse=query.reverse,
                                     limit=query.limit or None)
        return elements

    def _get_slice(self, elements: IResultSet, query: IResultSet) -> Iterable:
        """Get slice defined by `query.limit` and `query.offset`.

        :returns: IResultSet if not `query.limit`, else [IResource]
        """
        elements_slice = elements
        if query.limit:
            docids_slice = islice(elements.all(resolve=None),
                                  query.offset,
                                  query.offset + query.limit)
            elements_slice = [elements.resolver(x) for x in docids_slice]
        return elements_slice

    def _resolve(self, elements: Iterable, query: SearchQuery) -> Iterable:
        if query.resolve:
            elements = [x for x in elements]
        return elements

    def get_index(self, name) -> IIndex:
        system = self.get('system', {})
        adhocracy = self.get('adhocracy', {})
        index = system.get(name, None) or adhocracy.get(name, None)
        return index

    def _get_query_value(self, query_parameter: tuple) -> object:
        if self._is_tuple_starting_with_comparator(query_parameter):
            return query_parameter[1]
        else:
            return query_parameter

    def _get_query_comparator(self, query_parameter: tuple) -> object:
        if self._is_tuple_starting_with_comparator(query_parameter):
            return query_parameter[0]
        else:
            return

    def _is_tuple_starting_with_comparator(self, parameter: tuple) -> bool:
        if not isinstance(parameter, tuple):
            return False
        elif len(parameter) != 2:
            return False
        elif parameter[0] in self._comparators:
            return True

    _comparators = set(FieldComparator.__members__) \
        .union(KeywordComparator.__members__) \
        .union(KeywordSequenceComparator.__members__) \
        .union(FieldSequenceComparator.__members__)

def add_catalogs_system_and_adhocracy(context: ICatalogsService,
                                      registry: Registry,
                                      options: dict):
    """Add catalogs 'system' and 'adhocracy'."""
    context.add_catalog('system')
    context.add_catalog('adhocracy')

catalogs_service_meta = service_meta._replace(
    iresource=ICatalogsService,
    content_name='catalogs',
    content_class=CatalogsServiceAdhocracy,
    after_creation=[add_catalogs_system_and_adhocracy]
)

def includeme(config):
    """Register catalog utilities."""
    config.include('adhocracy_core.events')
    config.add_view_predicate('catalogable', catalog._CatalogablePredicate)
    config.add_directive('add_catalog_factory', catalog.add_catalog_factory)
    config.add_directive('add_indexview',
                         catalog.add_indexview,
                         action_wrap=False)
    config.registry.registerAdapter(catalog.deferred.BasicActionProcessor,
                                    (Interface,),
                                    IIndexingActionProcessor)
    add_resource_type_to_registry(catalogs_service_meta, config)
    config.scan('substanced.catalog')
    config.scan('.index')
    config.include('.adhocracy')
    config.include('.subscriber')
<|endoftext|>


<commit_before>from typing import Counter, Dict

import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

from lexos.helpers import definitions
from lexos.models.base_model import BaseModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.receivers.matrix_receiver import MatrixOption, MatrixReceiver, \
    IdTempLabelMap

FileIDContentMap = Dict[int, str]

class MatrixModel(BaseModel):

    def __init__(self, test_matrix_option: MatrixOption = None,
                 test_file_id_content_map: FileIDContentMap = None):
        """Class to generate and manipulate dtm.

        :param test_file_id_content_map: (fake parameter)
                                a map from file id to file contents
        :param test_matrix_option: (fake parameter)
                                the matrix options used for testing
        """
        super().__init__()
        self._test_file_id_content_map = test_file_id_content_map
        self._test_matrix_option = test_matrix_option

    @property
    def _file_id_content_map(self) -> FileIDContentMap:
        """Result form higher level class: the file manager of current session.

        :return: a file manager object
        """
        return self._test_file_id_content_map \
            if self._test_file_id_content_map \
            else FileManagerModel().load_file_manager() \
            .get_content_of_active_with_id()

    @property
    def _opts(self) -> MatrixOption:
        """Get all the options to use

        :return: either a frontend option or a fake option used for testing
        """
        return self._test_matrix_option if self._test_matrix_option \
            else MatrixReceiver().options_from_front_end()

    def get_temp_label(self) -> Counter[str]:
        """An unordered list (counter) of all the temp labels"""
        return Counter(self._opts.id_temp_label_map.values())

    def get_temp_label_id_map(self) -> IdTempLabelMap:
        """Get the dict where id maps to temp labels."""
        return self._opts.id_temp_label_map

    def _get_raw_count_matrix(self) -> pd.DataFrame:
        """Get the raw count matrix for the whole corpus

        :return: a panda data frame
            where
                - the index header is the file id
                - the row header is the words in the file
        """
        all_contents_with_id = self._file_id_content_map

        # a pair of parallel array
        file_ids = all_contents_with_id.keys()
        file_contents = all_contents_with_id.values()

        # heavy hitting tokenization and counting options set here

        # CountVectorizer can do
        #       (a) preprocessing
        #           (but we don't need that);
        #       (b) tokenization:
        #               analyzer=['word', 'char', or 'char_wb';
        #               Note: char_wb does not span across two words,
        #                   but will include whitespace at start/end of ngrams)
        #                   not an option in UI]
        #               token_pattern (only for analyzer='word'):
        #               cheng magic regex:
        #                   words include only NON-space characters
        #               ngram_range
        #                   (presuming this works for both word and char??)
        #       (c) culling:
        #           min_df..max_df
        #           (keep if term occurs in at least these documents)
        #       (d) stop_words handled in scrubber
        #       (e) lowercase=False (we want to leave the case as it is)
        #       (f) dtype=float
        #           sets type of resulting matrix of values;
        #           need float in case we use proportions

        # for example:
        # word 1-grams
        #   ['content' means use strings of text,
        #   analyzer='word' means features are "words";
        # min_df=1 means include word if it appears in at least one doc, the
        # default;

        # [\S]+  :
        #   means tokenize on a word boundary where boundary are \s
        #   (spaces, tabs, newlines)

        count_vector = CountVectorizer(
            input='content', encoding='utf-8', min_df=1,
            analyzer=self._opts.token_option.token_type,
            token_pattern=definitions.WORD_REGEX, lowercase=False,
            ngram_range=(self._opts.token_option.n_gram_size,
                         self._opts.token_option.n_gram_size),
            stop_words=[], dtype=float, max_df=1.0
        )

        # make a (sparse) Document-Term-Matrix (DTM) to hold all counts
        doc_term_sparse_matrix = count_vector.fit_transform(file_contents)

        # need to get at the entire matrix and not sparse matrix
        raw_count_matrix = doc_term_sparse_matrix.toarray()
        # snag all features (e.g., word-grams or char-grams) that were counted
        words = count_vector.get_feature_names()
        # pack the data into a data frame
        return pd.DataFrame(data=raw_count_matrix,
                            index=file_ids,
                            columns=words)

    def _apply_transformations_to_matrix(self, dtm_data_frame) -> pd.DataFrame:
        """Apply all the transitions to the matrix

        Currently there are following transitions with following order:
        - culling
        - most frequent word
        - tf-idf transformation
        - convert to proportion
        :param dtm_data_frame: the initial raw count data frame.
        :return: the final data frame after all the transformation
        """

        # apply culling to dtm
        if self._opts.culling_option.culling:

            dtm_data_frame = self._get_culled_matrix(
                least_num_seg=self._opts.culling_option.cull_least_passage,
                dtm_data_frame=dtm_data_frame
            )

        # only leaves the most frequent words in dtm
        if self._opts.culling_option.most_frequent_word:

            dtm_data_frame = self._get_most_frequent_word(
                lower_rank_bound=self._opts.culling_option.mfw_lowest_rank,
                dtm_data_frame=dtm_data_frame,
            )

        # ==== Parameters TfidfTransformer (TF/IDF) ===

        # Note: by default, idf use natural log
        #
        # (a) norm: 'l1', 'l2' or None, optional
        #     {USED AS THE LAST STEP: after getting the result of tf*idf,
        #       normalize the vector (row-wise) into unit vector}
        #     l1': Taxicab / Manhattan distance (p=1)
        #          [ ||u|| = |u1| + |u2| + |u3| ... ]
        #     l2': Euclidean norm (p=2), the most common norm;
        #           typically called "magnitude"
        #           [ ||u|| = sqrt( (u1)^2 + (u2)^2 + (u3)^2 + ... )]
        #     *** user can choose the normalization method ***
        #
        # (b) use_idf:
        #       boolean, optional ;
        #       "Enable inverse-document-frequency reweighting."
        #           which means: True if you want to use idf (times idf)
        #       False if you don't want to use idf at all,
        #           the result is only term-frequency
        #       *** we choose True here because the user has already chosen
        #           TF/IDF, instead of raw counts ***
        #
        # (c) smooth_idf:
        #       boolean, optional;
        #       "Smooth idf weights by adding one to document frequencies,
        #           as if an extra
        #       document was seen containing every term in the collection
        #            exactly once. Prevents zero divisions.""
        #       if True,
        #           idf = log(number of doc in total /
        #                       number of doc where term t appears) + 1
        #       if False,
        #           idf = log(number of doc in total + 1 /
        #                       number of doc where term t appears + 1 ) + 1
        #       *** we choose False, because denominator never equals 0
        #           in our case, no need to prevent zero divisions ***
        #
        # (d) sublinear_tf:
        #       boolean, optional ; "Apply sublinear tf scaling"
        #       if True,  tf = 1 + log(tf) (log here is base 10)
        #       if False, tf = term-frequency
        #       *** we choose False as the normal term-frequency ***

        if self._opts.norm_option.use_tf_idf:  # if use TF/IDF
            transformer = TfidfTransformer(
                norm=self._opts.norm_option.tf_idf_norm_option,
                use_idf=True,
                smooth_idf=False,
                sublinear_tf=False)
            dtm_data_frame = transformer.fit_transform(dtm_data_frame)

        # change the dtm to proportion
        if self._opts.norm_option.use_freq:
            # apply the proportion function to each row
            dtm_data_frame = dtm_data_frame.apply(lambda row: row / row.sum(),
                                                  axis=1)

        return dtm_data_frame

    def get_matrix(self)-> pd.DataFrame:
        """Get the document term matrix (DTM) of all the active files

        :return:
            a panda data frame with:
            - the index (row) header are file ids
            - the column header are words
        """

        raw_count_matrix = self._get_raw_count_matrix()

        return self._apply_transformations_to_matrix(raw_count_matrix)

    @staticmethod
    def _get_most_frequent_word(lower_rank_bound: int,
                                dtm_data_frame: pd.DataFrame) -> pd.DataFrame:
        """ Gets the most frequent words in final_matrix and words.

        The new count matrix will consists of only the most frequent words in
        the whole corpus.
        :param lower_rank_bound: the lowest rank to remain in the matrix
                                 (the rank is determined by the word's number
                                 of appearance in the whole corpus)
                                 (ranked from high to low)
        :param dtm_data_frame: the dtm in the form of panda data frame.
                                the indices(rows) are segment names
                                the columns are words.
        :return:
            dtm data frame with only the most frequent words
        """

        # get the word counts for each word in the entire corpus (1D array)
        word_count_in_corpus = dtm_data_frame.sum(axis='index')

        # get the index to sort those words
        sort_index_array = word_count_in_corpus.argsort()

        # get the total number of unique words
        total_num_words = word_count_in_corpus.size

        # strip the index to leave the most frequent ones
        # those are the index of the most frequent words
        most_frequent_index = sort_index_array[
            total_num_words - lower_rank_bound, lower_rank_bound]

        # use the most frequent index to get out most frequent words
        # this feature is called index array:
        # https://docs.scipy.org/doc/numpy/user/basics.indexing.html
        dtm_data_frame = dtm_data_frame.iloc[most_frequent_index]

        return dtm_data_frame

    @staticmethod
    def _get_culled_matrix(least_num_seg: int,
                           dtm_data_frame: pd.DataFrame) -> pd.DataFrame:
        """Gets the culled final_matrix and culled words.

        Gives a matrix that only contains the words that appears in more than
        `least_num_seg` segments.
        :param least_num_seg: least number of segment the word needs to appear
                                in to be kept.
        :param dtm_data_frame: the dtm in forms of panda data frames.
                                the indices(rows) are segment names
                                the columns are words.
        :return:
             the culled dtm data frame
        """

        # create a bool matrix to indicate whether a word is in a segment
        # at the line of segment s and the column of word w,
        # if the value is True, then means w is in s
        # otherwise means w is not in s
        is_in_data_frame = dtm_data_frame.astype(bool)

        # summing the boolean array gives an int, which indicates how many
        # True there are in that array.
        # this is an series, indicating each word is in how many segments
        # this array is a parallel array of words
        # noinspection PyUnresolvedReferences
        words_in_num_seg_series = is_in_data_frame.sum(axis=0)

        # get the index of all the words needs to remain
        # this is an array of int
        dtm_data_frame = dtm_data_frame.loc[
            words_in_num_seg_series >= least_num_seg
            ]

        return dtm_data_frame
<commit_msg>make the properties more strict<commit_after>from typing import Counter, Dict

import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

from lexos.helpers import definitions
from lexos.models.base_model import BaseModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.receivers.matrix_receiver import MatrixOption, MatrixReceiver, \
    IdTempLabelMap

FileIDContentMap = Dict[int, str]

class MatrixModel(BaseModel):

    def __init__(self, test_matrix_option: MatrixOption = None,
                 test_file_id_content_map: FileIDContentMap = None):
        """Class to generate and manipulate dtm.

        :param test_file_id_content_map: (fake parameter)
                                a map from file id to file contents
        :param test_matrix_option: (fake parameter)
                                the matrix options used for testing
        """
        super().__init__()
        self._test_file_id_content_map = test_file_id_content_map
        self._test_matrix_option = test_matrix_option

    @property
    def _file_id_content_map(self) -> FileIDContentMap:
        """Result form higher level class: the file manager of current session.

        :return: a file manager object
        """
        return self._test_file_id_content_map \
            if self._test_file_id_content_map is not None \
            else FileManagerModel().load_file_manager() \
            .get_content_of_active_with_id()

    @property
    def _opts(self) -> MatrixOption:
        """Get all the options to use

        :return: either a frontend option or a fake option used for testing
        """
        return self._test_matrix_option \
            if self._test_matrix_option is not None \
            else MatrixReceiver().options_from_front_end()

    def get_temp_label(self) -> Counter[str]:
        """An unordered list (counter) of all the temp labels"""
        return Counter(self._opts.id_temp_label_map.values())

    def get_temp_label_id_map(self) -> IdTempLabelMap:
        """Get the dict where id maps to temp labels."""
        return self._opts.id_temp_label_map

    def _get_raw_count_matrix(self) -> pd.DataFrame:
        """Get the raw count matrix for the whole corpus

        :return: a panda data frame
            where
                - the index header is the file id
                - the row header is the words in the file
        """
        all_contents_with_id = self._file_id_content_map

        # a pair of parallel array
        file_ids = all_contents_with_id.keys()
        file_contents = all_contents_with_id.values()

        # heavy hitting tokenization and counting options set here

        # CountVectorizer can do
        #       (a) preprocessing
        #           (but we don't need that);
        #       (b) tokenization:
        #               analyzer=['word', 'char', or 'char_wb';
        #               Note: char_wb does not span across two words,
        #                   but will include whitespace at start/end of ngrams)
        #                   not an option in UI]
        #               token_pattern (only for analyzer='word'):
        #               cheng magic regex:
        #                   words include only NON-space characters
        #               ngram_range
        #                   (presuming this works for both word and char??)
        #       (c) culling:
        #           min_df..max_df
        #           (keep if term occurs in at least these documents)
        #       (d) stop_words handled in scrubber
        #       (e) lowercase=False (we want to leave the case as it is)
        #       (f) dtype=float
        #           sets type of resulting matrix of values;
        #           need float in case we use proportions

        # for example:
        # word 1-grams
        #   ['content' means use strings of text,
        #   analyzer='word' means features are "words";
        # min_df=1 means include word if it appears in at least one doc, the
        # default;

        # [\S]+  :
        #   means tokenize on a word boundary where boundary are \s
        #   (spaces, tabs, newlines)

        count_vector = CountVectorizer(
            input='content', encoding='utf-8', min_df=1,
            analyzer=self._opts.token_option.token_type,
            token_pattern=definitions.WORD_REGEX, lowercase=False,
            ngram_range=(self._opts.token_option.n_gram_size,
                         self._opts.token_option.n_gram_size),
            stop_words=[], dtype=float, max_df=1.0
        )

        # make a (sparse) Document-Term-Matrix (DTM) to hold all counts
        doc_term_sparse_matrix = count_vector.fit_transform(file_contents)

        # need to get at the entire matrix and not sparse matrix
        raw_count_matrix = doc_term_sparse_matrix.toarray()
        # snag all features (e.g., word-grams or char-grams) that were counted
        words = count_vector.get_feature_names()
        # pack the data into a data frame
        return pd.DataFrame(data=raw_count_matrix,
                            index=file_ids,
                            columns=words)

    def _apply_transformations_to_matrix(self, dtm_data_frame) -> pd.DataFrame:
        """Apply all the transitions to the matrix

        Currently there are following transitions with following order:
        - culling
        - most frequent word
        - tf-idf transformation
        - convert to proportion
        :param dtm_data_frame: the initial raw count data frame.
        :return: the final data frame after all the transformation
        """

        # apply culling to dtm
        if self._opts.culling_option.culling:

            dtm_data_frame = self._get_culled_matrix(
                least_num_seg=self._opts.culling_option.cull_least_passage,
                dtm_data_frame=dtm_data_frame
            )

        # only leaves the most frequent words in dtm
        if self._opts.culling_option.most_frequent_word:

            dtm_data_frame = self._get_most_frequent_word(
                lower_rank_bound=self._opts.culling_option.mfw_lowest_rank,
                dtm_data_frame=dtm_data_frame,
            )

        # ==== Parameters TfidfTransformer (TF/IDF) ===

        # Note: by default, idf use natural log
        #
        # (a) norm: 'l1', 'l2' or None, optional
        #     {USED AS THE LAST STEP: after getting the result of tf*idf,
        #       normalize the vector (row-wise) into unit vector}
        #     l1': Taxicab / Manhattan distance (p=1)
        #          [ ||u|| = |u1| + |u2| + |u3| ... ]
        #     l2': Euclidean norm (p=2), the most common norm;
        #           typically called "magnitude"
        #           [ ||u|| = sqrt( (u1)^2 + (u2)^2 + (u3)^2 + ... )]
        #     *** user can choose the normalization method ***
        #
        # (b) use_idf:
        #       boolean, optional ;
        #       "Enable inverse-document-frequency reweighting."
        #           which means: True if you want to use idf (times idf)
        #       False if you don't want to use idf at all,
        #           the result is only term-frequency
        #       *** we choose True here because the user has already chosen
        #           TF/IDF, instead of raw counts ***
        #
        # (c) smooth_idf:
        #       boolean, optional;
        #       "Smooth idf weights by adding one to document frequencies,
        #           as if an extra
        #       document was seen containing every term in the collection
        #            exactly once. Prevents zero divisions.""
        #       if True,
        #           idf = log(number of doc in total /
        #                       number of doc where term t appears) + 1
        #       if False,
        #           idf = log(number of doc in total + 1 /
        #                       number of doc where term t appears + 1 ) + 1
        #       *** we choose False, because denominator never equals 0
        #           in our case, no need to prevent zero divisions ***
        #
        # (d) sublinear_tf:
        #       boolean, optional ; "Apply sublinear tf scaling"
        #       if True,  tf = 1 + log(tf) (log here is base 10)
        #       if False, tf = term-frequency
        #       *** we choose False as the normal term-frequency ***

        if self._opts.norm_option.use_tf_idf:  # if use TF/IDF
            transformer = TfidfTransformer(
                norm=self._opts.norm_option.tf_idf_norm_option,
                use_idf=True,
                smooth_idf=False,
                sublinear_tf=False)
            dtm_data_frame = transformer.fit_transform(dtm_data_frame)

        # change the dtm to proportion
        if self._opts.norm_option.use_freq:
            # apply the proportion function to each row
            dtm_data_frame = dtm_data_frame.apply(lambda row: row / row.sum(),
                                                  axis=1)

        return dtm_data_frame

    def get_matrix(self)-> pd.DataFrame:
        """Get the document term matrix (DTM) of all the active files

        :return:
            a panda data frame with:
            - the index (row) header are file ids
            - the column header are words
        """

        raw_count_matrix = self._get_raw_count_matrix()

        return self._apply_transformations_to_matrix(raw_count_matrix)

    @staticmethod
    def _get_most_frequent_word(lower_rank_bound: int,
                                dtm_data_frame: pd.DataFrame) -> pd.DataFrame:
        """ Gets the most frequent words in final_matrix and words.

        The new count matrix will consists of only the most frequent words in
        the whole corpus.
        :param lower_rank_bound: the lowest rank to remain in the matrix
                                 (the rank is determined by the word's number
                                 of appearance in the whole corpus)
                                 (ranked from high to low)
        :param dtm_data_frame: the dtm in the form of panda data frame.
                                the indices(rows) are segment names
                                the columns are words.
        :return:
            dtm data frame with only the most frequent words
        """

        # get the word counts for each word in the entire corpus (1D array)
        word_count_in_corpus = dtm_data_frame.sum(axis='index')

        # get the index to sort those words
        sort_index_array = word_count_in_corpus.argsort()

        # get the total number of unique words
        total_num_words = word_count_in_corpus.size

        # strip the index to leave the most frequent ones
        # those are the index of the most frequent words
        most_frequent_index = sort_index_array[
            total_num_words - lower_rank_bound, lower_rank_bound]

        # use the most frequent index to get out most frequent words
        # this feature is called index array:
        # https://docs.scipy.org/doc/numpy/user/basics.indexing.html
        dtm_data_frame = dtm_data_frame.iloc[most_frequent_index]

        return dtm_data_frame

    @staticmethod
    def _get_culled_matrix(least_num_seg: int,
                           dtm_data_frame: pd.DataFrame) -> pd.DataFrame:
        """Gets the culled final_matrix and culled words.

        Gives a matrix that only contains the words that appears in more than
        `least_num_seg` segments.
        :param least_num_seg: least number of segment the word needs to appear
                                in to be kept.
        :param dtm_data_frame: the dtm in forms of panda data frames.
                                the indices(rows) are segment names
                                the columns are words.
        :return:
             the culled dtm data frame
        """

        # create a bool matrix to indicate whether a word is in a segment
        # at the line of segment s and the column of word w,
        # if the value is True, then means w is in s
        # otherwise means w is not in s
        is_in_data_frame = dtm_data_frame.astype(bool)

        # summing the boolean array gives an int, which indicates how many
        # True there are in that array.
        # this is an series, indicating each word is in how many segments
        # this array is a parallel array of words
        # noinspection PyUnresolvedReferences
        words_in_num_seg_series = is_in_data_frame.sum(axis=0)

        # get the index of all the words needs to remain
        # this is an array of int
        dtm_data_frame = dtm_data_frame.loc[
            words_in_num_seg_series >= least_num_seg
            ]

        return dtm_data_frame
<|endoftext|>


<commit_before>"""Record Nest API data to InfluxDB.

The ``record`` API is designed to persist Nest thermostat data in an InfluxDB database.

"""

from contextlib import closing
from urllib import urlencode
from urlparse import SplitResult, urlunsplit

import json
import logging
import os

from influxdb import client as influxdb

import requests

NEST_API_PROTOCOL = "https"
NEST_API_LOCATION = "developer-api.nest.com"
NEST_API_ACCESS_TOKEN = os.environ["DEN_ACCESS_TOKEN"]
"""The Nest API `access token
<https://developer.nest.com/documentation/cloud/how-to-auth#exchange-your-authorization-code-for-an-access-token>`_.

.. note::

   This value is taken from the environment variable ``DEN_ACCESS_TOKEN``.  It is required to authenticate with the
   Nest API.

"""

def _get_api_url(path=""):
    """Get a Nest API URL for the given path."""
    query = urlencode({"auth": NEST_API_ACCESS_TOKEN})
    split = SplitResult(scheme=NEST_API_PROTOCOL, netloc=NEST_API_LOCATION, path=path.strip("/"), query=query,
                        fragment="")
    return urlunsplit(split)

def _get_stream(path=""):
    """Make a GET request to the Nest REST stream API and return the response object."""
    r = requests.get(_get_api_url(path), headers={"Accept": "text/event-stream"}, stream=True)
    for h in r.history:
        logging.debug("[%d] Redirect: %s", h.status_code, h.url)
    logging.debug("[%d] URL: %s", r.status_code, r.url)
    r.raise_for_status()
    return r

def _is_event(line):
    """Is the given line an event line?"""
    return line.startswith("event:")

def _is_data(line):
    """Is the given line a data line?"""
    return line.startswith("data:")

def _process_event(line):
    """Process the given event line."""
    _, event = line.split(":", 1)
    event = event.strip()
    logging.debug(event)
    return None if event == "keep-alive" else event

def _process_data(line):
    """Process the given data line."""
    _, data_str = line.split(":", 1)
    return json.loads(data_str.strip())

def _process(line):
    """Process the given line."""
    if _is_event(line):
        _process_event(line)
    return _process_data(line) if _is_data(line) else None

def _get_structures(data):
    """Get structure data from the given data dict."""
    structures = data["data"]["structures"]
    return [structures[s] for s in structures]

def _get_structure_data(data):
    """Get structure data to write to InfluxDB."""
    name = "structures"
    structures = _get_structures(data)
    points = []
    for s in structures:
        try:
            del s["thermostats"]
        except KeyError:
            pass
        points.append(s.values())
    columns = structures[0].keys()
    return [{"name": name, "columns": columns, "points": points}]

def _get_thermostats(data):
    """Get thermostat data from the given data dict."""
    thermostats = data["data"]["devices"]["thermostats"]
    return [thermostats[t] for t in thermostats]

def _get_thermostat_data(data):
    """Get thermostat data to write to InfluxDB."""
    name = "thermostats"
    thermostats = _get_thermostats(data)
    columns = thermostats[0].keys()
    points = [t.values() for t in thermostats]
    return [{"name": name, "columns": columns, "points": points}]

def configure_logging():
    """Configure basic logging."""
    logging.basicConfig(filename="den.log", level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s")

def record(database, port, ssl):
    """Stream results from the API and record them in the database."""
    db = influxdb.InfluxDBClient(database=database, port=port, ssl=ssl)
    with closing(_get_stream()) as stream:
        logging.debug("[%d] Streaming %s", stream.status_code, stream.url)
        for l in stream.iter_lines():
            if l:
                value = _process(l)
                if value:
                    logging.info(value)
                    db.write_points(_get_structure_data(value))
                    db.write_points(_get_thermostat_data(value))
        logging.debug("[%d] Streaming complete %s", stream.status_code, stream.url)
<commit_msg>record - Cleanup lint warnings.<commit_after>"""Record Nest API data to InfluxDB.

The ``record`` API is designed to persist Nest thermostat data in an InfluxDB database.

"""

from contextlib import closing
from urllib import urlencode
from urlparse import SplitResult, urlunsplit

import json
import logging
import os

from influxdb import client as influxdb

import requests

NEST_API_PROTOCOL = "https"
NEST_API_LOCATION = "developer-api.nest.com"
NEST_API_ACCESS_TOKEN = os.environ["DEN_ACCESS_TOKEN"]
"""The Nest API `access token
<https://developer.nest.com/documentation/cloud/how-to-auth#exchange-your-authorization-code-for-an-access-token>`_.

.. note::

   This value is taken from the environment variable ``DEN_ACCESS_TOKEN``.  It is required to authenticate with the
   Nest API.

"""

def _get_api_url(path=""):
    """Get a Nest API URL for the given path."""
    query = urlencode({"auth": NEST_API_ACCESS_TOKEN})
    split = SplitResult(scheme=NEST_API_PROTOCOL, netloc=NEST_API_LOCATION, path=path.strip("/"), query=query,
                        fragment="")
    return urlunsplit(split)

def _get_stream(path=""):
    """Make a GET request to the Nest REST stream API and return the response object."""
    r = requests.get(_get_api_url(path), headers={"Accept": "text/event-stream"}, stream=True)
    for h in r.history:
        logging.debug("[%d] Redirect: %s", h.status_code, h.url)
    logging.debug("[%d] URL: %s", r.status_code, r.url)
    r.raise_for_status()
    return r

def _is_event(line):
    """Determine if ``line`` represents an event."""
    return line.startswith("event:")

def _is_data(line):
    """Determine if ``line`` represents data."""
    return line.startswith("data:")

def _process_event(line):
    """Process the given event line."""
    _, event = line.split(":", 1)
    event = event.strip()
    logging.debug(event)
    return None if event == "keep-alive" else event

def _process_data(line):
    """Process the given data line."""
    _, data_str = line.split(":", 1)
    return json.loads(data_str.strip())

def _process(line):
    """Process the given line."""
    if _is_event(line):
        _process_event(line)
    return _process_data(line) if _is_data(line) else None

def _get_structures(data):
    """Get structure data from the given data dict."""
    structures = data["data"]["structures"]
    return [structures[s] for s in structures]

def _get_structure_data(data):
    """Get structure data to write to InfluxDB."""
    name = "structures"
    structures = _get_structures(data)
    points = []
    for s in structures:
        try:
            del s["thermostats"]
        except KeyError:  # pylint: disable=pointless-except
            pass
        points.append(s.values())
    columns = structures[0].keys()
    return [{"name": name, "columns": columns, "points": points}]

def _get_thermostats(data):
    """Get thermostat data from the given data dict."""
    thermostats = data["data"]["devices"]["thermostats"]
    return [thermostats[t] for t in thermostats]

def _get_thermostat_data(data):
    """Get thermostat data to write to InfluxDB."""
    name = "thermostats"
    thermostats = _get_thermostats(data)
    columns = thermostats[0].keys()
    points = [t.values() for t in thermostats]
    return [{"name": name, "columns": columns, "points": points}]

def configure_logging():
    """Configure basic logging."""
    logging.basicConfig(filename="den.log", level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s")

def record(database, port, ssl):
    """Stream results from the API and record them in the database."""
    db = influxdb.InfluxDBClient(database=database, port=port, ssl=ssl)
    with closing(_get_stream()) as stream:
        logging.debug("[%d] Streaming %s", stream.status_code, stream.url)
        for l in stream.iter_lines():
            if l:
                value = _process(l)
                if value:
                    logging.info(value)
                    db.write_points(_get_structure_data(value))
                    db.write_points(_get_thermostat_data(value))
        logging.debug("[%d] Streaming complete %s", stream.status_code, stream.url)
<|endoftext|>


<commit_before>import datetime
import inspect
from contextlib import contextmanager
from typing import Iterator, Callable, Any

from approvaltests.utilities.string_wrapper import StringWrapper
from approvaltests.namer import StackFrameNamer

class LoggingInstance:

    def __init__(self):
        self.previous_timestamp = None
        self.logger = lambda t: print(t, end="")
        self.tabbing = 0
        self.counter = 0
        self.log_with_timestamps = True
        self.timer: Callable[[], datetime.datetime] = datetime.datetime.now

    def log_to_string(self) -> StringWrapper:
        buffer = StringWrapper()
        self.logger = buffer.append
        self.log_with_timestamps = False
        return buffer

    @contextmanager
    def use_markers(self, additional_stack: int = 0) -> Iterator[None]:
        stack_position = 1 + additional_stack
        stack = inspect.stack(stack_position)[2]
        method_name = stack.function
        filename = StackFrameNamer.get_class_name_for_frame(stack)
        expected = f"-> in: {method_name}(){filename}"
        self.log_and_add_timestamps_if_needed(expected)
        self.tabbing = self.tabbing + 1
        yield
        self.tabbing = self.tabbing - 1
        expected = f"<- out: {method_name}(){filename}"
        self.log_and_add_timestamps_if_needed(expected)
        pass

    def log_and_add_timestamps_if_needed(self, text: str, use_timestamps=True) -> None:
        if self.counter != 0:
            self.logger("\n")
            self.counter = 0
        timestamp = self.get_timestamp() if use_timestamps else ""
        output_message = f"{timestamp}{self.get_tabs()}{text}\n"
        self.logger(output_message)

    def get_timestamp(self) -> str:
        timestamp = ""
        if self.log_with_timestamps:
            time1: datetime.datetime = self.timer()
            time = time1.strftime("%Y-%m-%dT%H:%M:%SZ")
            diff_millseconds = 0
            if self.previous_timestamp != None:
                delta = time1 - self.previous_timestamp
                diff_millseconds = int((delta).total_seconds() * 1000)
            diff = diff_millseconds
            diff_display = f" ~{diff:06}ms"
            timestamp = f"[{time} {diff_display}] "
            self.previous_timestamp = time1
        return timestamp

    def variable(self, name: str, value: Any) -> None:
        display_variable = f"variable: {name} = {value}"
        self.log_and_add_timestamps_if_needed(display_variable)

    def hour_glass(self) -> None:
        self.increment_hour_glass_counter()
        if self.counter == 1:
            self.logger(f"{self.get_tabs()}.")
        elif self.counter == 100:
            self.logger("10\n")
            self.counter = 0
        elif self.counter % 10 == 0:
            digit = int(self.counter / 10)
            self.logger(f"{digit}")
        else:
            self.logger(".")

    def get_tabs(self) -> str:
        return "  " * self.tabbing

    def increment_hour_glass_counter(self) -> None:
        self.counter = self.counter + 1

    def event(self, event_name: str) -> None:
        self.log_and_add_timestamps_if_needed(f"event: {event_name}")

    def query(self, query_text:str) -> None:
        self.log_and_add_timestamps_if_needed(f"Sql: {query_text}")

    def message(self, message):
        self.log_and_add_timestamps_if_needed(f"message: {message}")

    def warning(self, exception: Exception) -> None:
        warning_text = "*" * 91
        self.log_and_add_timestamps_if_needed(warning_text, use_timestamps=False)
        if self.log_with_timestamps:
            self.log_and_add_timestamps_if_needed("")
        self.log_and_add_timestamps_if_needed(str(exception), use_timestamps=False)
        self.log_and_add_timestamps_if_needed(warning_text, use_timestamps=False)<commit_msg>R rename<commit_after>import datetime
import inspect
from contextlib import contextmanager
from typing import Iterator, Callable, Any

from approvaltests.utilities.string_wrapper import StringWrapper
from approvaltests.namer import StackFrameNamer

class LoggingInstance:

    def __init__(self):
        self.previous_timestamp = None
        self.logger = lambda t: print(t, end="")
        self.tabbing = 0
        self.counter = 0
        self.log_with_timestamps = True
        self.timer: Callable[[], datetime.datetime] = datetime.datetime.now

    def log_to_string(self) -> StringWrapper:
        buffer = StringWrapper()
        self.logger = buffer.append
        self.log_with_timestamps = False
        return buffer

    @contextmanager
    def use_markers(self, additional_stack: int = 0) -> Iterator[None]:
        stack_position = 1 + additional_stack
        stack = inspect.stack(stack_position)[2]
        method_name = stack.function
        filename = StackFrameNamer.get_class_name_for_frame(stack)
        expected = f"-> in: {method_name}(){filename}"
        self.log_line(expected)
        self.tabbing = self.tabbing + 1
        yield
        self.tabbing = self.tabbing - 1
        expected = f"<- out: {method_name}(){filename}"
        self.log_line(expected)
        pass

    def log_line(self, text: str, use_timestamps=True) -> None:
        if self.counter != 0:
            self.logger("\n")
            self.counter = 0
        timestamp = self.get_timestamp() if use_timestamps else ""
        output_message = f"{timestamp}{self.get_tabs()}{text}\n"
        self.logger(output_message)

    def get_timestamp(self) -> str:
        timestamp = ""
        if self.log_with_timestamps:
            time1: datetime.datetime = self.timer()
            time = time1.strftime("%Y-%m-%dT%H:%M:%SZ")
            diff_millseconds = 0
            if self.previous_timestamp != None:
                delta = time1 - self.previous_timestamp
                diff_millseconds = int((delta).total_seconds() * 1000)
            diff = diff_millseconds
            diff_display = f" ~{diff:06}ms"
            timestamp = f"[{time} {diff_display}] "
            self.previous_timestamp = time1
        return timestamp

    def variable(self, name: str, value: Any) -> None:
        display_variable = f"variable: {name} = {value}"
        self.log_line(display_variable)

    def hour_glass(self) -> None:
        self.increment_hour_glass_counter()
        if self.counter == 1:
            self.logger(f"{self.get_tabs()}.")
        elif self.counter == 100:
            self.logger("10\n")
            self.counter = 0
        elif self.counter % 10 == 0:
            digit = int(self.counter / 10)
            self.logger(f"{digit}")
        else:
            self.logger(".")

    def get_tabs(self) -> str:
        return "  " * self.tabbing

    def increment_hour_glass_counter(self) -> None:
        self.counter = self.counter + 1

    def event(self, event_name: str) -> None:
        self.log_line(f"event: {event_name}")

    def query(self, query_text:str) -> None:
        self.log_line(f"Sql: {query_text}")

    def message(self, message):
        self.log_line(f"message: {message}")

    def warning(self, exception: Exception) -> None:
        warning_text = "*" * 91
        self.log_line(warning_text, use_timestamps=False)
        if self.log_with_timestamps:
            self.log_line("", use_timestamps=True)
        self.log_line(str(exception), use_timestamps=False)
        self.log_line(warning_text, use_timestamps=False)<|endoftext|>


<commit_before>#!/usr/bin/env python
import time
import requests  # pip install requests
import logging

def autorunPhotoImportScript(opts):
    sleepSeconds = 120
    timeoutSeconds = 20
    while True:
        scriptUrl = 'http://%s/fileUpdate.lua' % opts.hostname
        r = requests.get(scriptUrl, timeout=timeoutSeconds)
        logging.info(r.text)
        time.sleep(sleepSeconds)

def main():
    import optparse
    parser = optparse.OptionParser('usage: %prog')
    parser.add_option('-n', '--hostname',
                      default="",
                      help='name of the subsystem to ping')
    opts, _args = parser.parse_args()
    autorunPhotoImportScript(opts)

if __name__ == '__main__':
    main()<commit_msg>autorun script<commit_after>#!/usr/bin/env python
import time
import requests  # pip install requests
import logging

def autorunPhotoImportScript(opts):
    sleepSeconds = 120
    timeoutSeconds = 20
    while True:
        scriptUrl = 'http://%s/fileUpdate.lua' % opts.hostname
        r = requests.get(scriptUrl, timeout=timeoutSeconds)
        logging.info(r.text)
        time.sleep(sleepSeconds)

def main():
    import optparse
    parser = optparse.OptionParser('usage: %prog')
    parser.add_option('-n', '--hostname',
                      default="",
                      help='name of the subsystem to ping')
    opts, _args = parser.parse_args()
    autorunPhotoImportScript(opts)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of Pate, Kate' Python scripting plugin.
#
# Copyright (C) 2006 Paul Giannaros <paul@giannaros.org>
# Copyright (C) 2013 Shaheed Haque <srhaque@theiet.org>
# Copyright (C) 2013 Alex Turbov <i.zaufi@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) version 3.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; see the file COPYING.LIB.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.

'''Decorators used in plugins'''

import functools
import inspect
import sys
import traceback
import xml.dom.minidom

from PyQt4 import QtCore, QtGui
from PyKDE4 import kdecore, kdeui
from PyKDE4.kdecore import i18nc

import pate

from .api import *

_registered_xml_gui_clients = dict()

def getXmlGuiClient(plugin=None, ef=1, use_inspect=False):
    '''Provide an access to an XML GUI client for a current plugin'''
    if plugin is None:
        if use_inspect:
            plugin = inspect.getmoduleinfo(inspect.stack()[ef + 1][1])[0]
        else:
            plugin = sys._getframe(1).f_globals['__name__']

    kDebug('Getting XMLGUIClient for {}/{}/{}'.format(plugin, ef, use_inspect))
    if plugin in _registered_xml_gui_clients:
        return _registered_xml_gui_clients[plugin]

#
# initialization related stuff
#

def pateEventHandler(event):
    def _decorator(func):
        setattr(pate, event, func)
        del func
    return _decorator

def _callAll(plugin, functions, *args, **kwargs):
    if plugin in functions:
        for f in functions[plugin]:
            try:
                f(*args, **kwargs)
            except:
                traceback.print_exc()
                sys.stderr.write('\n')
                # TODO Return smth to a caller, so in case of
                # failed initialization it may report smth to the
                # C++ level and latter can show an error to the user...
                continue

def _simpleEventListener(func):
    # automates the most common decorator pattern: calling a bunch
    # of functions when an event has occurred
    func.functions = dict()
    func.fire = functools.partial(_callAll, functions=func.functions)
    func.clear = func.functions.clear
    return func

def _registerCallback(plugin, event, func):
    if plugin not in event.functions:
        event.functions[plugin] = set()

    event.functions[plugin].add(func)
    return func

@_simpleEventListener
def init(func):
    ''' The function will be called when particular plugin has loaded
        and the configuration has been initiated
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    kDebug('@init: {}/{}'.format(plugin, func.__name__))
    return _registerCallback(plugin, init, func)

@_simpleEventListener
def unload(func):
    ''' The function will be called when particular plugin is being
        unloaded from memory. Clean up any widgets that you have added
        to the interface (toolviews etc).

        ATTENTION Be really careful trying to access any window, view
            or document from the @unload handler: in case of application
            quit everything is dead already!
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    kDebug('@unload: {}/{}'.format(plugin, func.__name__))
    def _module_cleaner():
        kDebug('@unload/cleaner: {}/{}'.format(plugin, func.__name__))
        if plugin in _registered_xml_gui_clients and mainInterfaceWindow() is not None:
            clnt = _registered_xml_gui_clients[plugin]
            clnt.actionCollection().clearAssociatedWidgets()
            clnt.actionCollection().clear()
            mainInterfaceWindow().guiFactory().removeClient(clnt)
            del _registered_xml_gui_clients[plugin]

        if plugin in init.functions:
            kDebug('@unload/init-cleaner: {}/{}'.format(plugin, func.__name__))
            del init.functions[plugin]

        func()

    return _registerCallback(plugin, unload, _module_cleaner)

@_simpleEventListener
def viewCreated(func):
    ''' Calls the function when a new view is created, passing the
        view as a parameter.
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    mainWindow = application.activeMainWindow()
    assert(mainWindow is not None)
    mainWindow.connect(mainWindow, QtCore.SIGNAL('viewCreated(KTextEditor::View*)'), func)
    return func

@_simpleEventListener
def viewChanged(func):
    ''' Calls the function when the view changes.
        To access the new active view, use kate.activeView()

        NOTE The very first call after kate starts the view actually is None!
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    mainWindow = application.activeMainWindow()
    assert(mainWindow is not None)
    mainWindow.connect(mainWindow, QtCore.SIGNAL('viewChanged()'), func)
    return func

def configPage(name, fullName, icon):
    ''' Decorator that adds a configPage into Kate's settings dialog. When the
    item is fired, your function is called.

    Parameters:
        * name -        The text associated with the configPage in the list of
                        config pages.
        * fullName -    The title of the configPage when selected.
        * icon -        An icon to associate with this configPage. Pass a string
                        to use KDE's image loading system or a QPixmap or
                        QIcon to use any custom icon.

    NOTE: Kate may need to be restarted for this decorator to take effect, or
    to remove all traces of the plugin on removal.
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    def _decorator(func):
        a = name, fullName, kdeui.KIcon(icon)
        func.configPage = a
        return func
    return _decorator

def moduleGetConfigPages(module):
    '''Return a list of each module function decorated with @configPage.

    The returned object is [ { function, callable, ( name, fullName, icon ) }... ].
    '''
    result = []
    for k, v in module.__dict__.items():
        if inspect.isfunction(v) and hasattr(v, 'configPage'):
            result.append((k, v, v.configPage))
    return result

#
# Actions related stuff
#

class _HandledException(Exception):
    def __init__(self, message):
        super(_HandledException, self).__init__(message)

class _catchAllHandler(object):
    '''Standard error handling for plugin actions.'''
    def __init__(self, f):
        self.f = f

    def __call__(self):
        try:
            return self.f()
        except _HandledException:
            raise
        except Exception as e:
            txt = ''.join(traceback.format_exception(*sys.exc_info()))
            KMessageBox.error(
                None
              , txt
              , i18nc('@title:window', 'Error in action <icode>%1</icode>', self.f.__name__)
              )
            raise _HandledException(txt)

class InvalidAction(Exception):
    pass

def action(func):
    ''' Decorator that adds an action to the menu bar. When the item is fired,
        your function is called
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    kDebug('@action: {}/{}'.format(plugin, func.__name__))
    ui_file = kdecore.KGlobal.dirs().findResource('appdata', 'pate/{}_ui.rc'.format(plugin))
    if not ui_file:
        ui_file = kdecore.KGlobal.dirs().findResource('appdata', 'pate/{}/{}_ui.rc'.format(plugin, plugin))
        if not ui_file:
            # TODO Report an error: found an action w/o corresponding ui.rc file!
            return func

    # Found UI resource file
    #kDebug('ui_file={}'.format(repr(ui_file)))

    # Get the XML GUI client or create a new one
    clnt = None
    if plugin not in _registered_xml_gui_clients:
        kDebug('@action: make XMLGUIClient for plugin={}'.format(plugin))
        clnt = kdeui.KXMLGUIClient()
        clnt.replaceXMLFile(ui_file,ui_file)
    else:
        clnt = _registered_xml_gui_clients[plugin]
        mainInterfaceWindow().guiFactory().removeClient(clnt)

    # Find an action details
    gui = xml.dom.minidom.parse(ui_file)
    found = False
    for tag in gui.getElementsByTagName('Action'):
        # Action at least must have a name and text
        # and name must be the same as a function name marked w/ @action decorator
        want_it = 'name' in tag.attributes \
            and 'text' in tag.attributes \
            and func.__name__ == tag.attributes['name'].value
        if want_it:
            found = True
            # Get mandatory attributes
            name = tag.attributes['name'].value
            text = tag.attributes['text'].value
            act = clnt.actionCollection().addAction(name)
            act.setText(text)
            kDebug('@action/found: {} --> "{}"'.format(name, text))
            # Get optional attributes
            if 'shortcut' in tag.attributes:
                act.setShortcut(QtGui.QKeySequence(tag.attributes['shortcut'].value))
            if 'icon' in tag.attributes:
                act.setIcon(kdeui.KIcon(tag.attributes['icon'].value))
            if 'iconText' in tag.attributes:
                act.setIconText(tag.attributes['iconText'].value)
            if 'toolTip' in tag.attributes:
                act.setToolTip(tag.attributes['toolTip'].value)
            if 'whatsThis' in tag.attributes:
                act.setWhatsThis(tag.attributes['whatsThis'].value)
            # Connect it to the function
            act.connect(act, QtCore.SIGNAL('triggered()'), func)
            # Make an action accessible from the targeted function as its attribute
            setattr(func, 'action', act)
            # No need to scan anything else!!!
            # ATTENTION It is why the very first action declaration
            # MUST contain everything! While others, w/ same name,
            # may have only name! For example, if same action must be
            # added to toolbar or context menu...
            break

    if not found:
        raise InvalidAction(
            i18nc(
                '@info:tooltip'
              , 'Invalid <filename>%1</filename> file: No action with name <icode>%2</icode> defined'
              , ui_file
              , func.__name__
              )
          )

    if plugin not in _registered_xml_gui_clients:
        _registered_xml_gui_clients[plugin] = clnt

    mainInterfaceWindow().guiFactory().addClient(clnt)

    return func
<commit_msg>fix KeyError in actions decorator on Python 2<commit_after># -*- coding: utf-8 -*-
# This file is part of Pate, Kate' Python scripting plugin.
#
# Copyright (C) 2006 Paul Giannaros <paul@giannaros.org>
# Copyright (C) 2013 Shaheed Haque <srhaque@theiet.org>
# Copyright (C) 2013 Alex Turbov <i.zaufi@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) version 3.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; see the file COPYING.LIB.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.

'''Decorators used in plugins'''

import functools
import inspect
import sys
import traceback
import xml.dom.minidom

from PyQt4 import QtCore, QtGui
from PyKDE4 import kdecore, kdeui
from PyKDE4.kdecore import i18nc

import pate

from .api import *

_registered_xml_gui_clients = dict()

def getXmlGuiClient(plugin=None, ef=1, use_inspect=False):
    '''Provide an access to an XML GUI client for a current plugin'''
    if plugin is None:
        if use_inspect:
            plugin = inspect.getmoduleinfo(inspect.stack()[ef + 1][1])[0]
        else:
            plugin = sys._getframe(1).f_globals['__name__']

    kDebug('Getting XMLGUIClient for {}/{}/{}'.format(plugin, ef, use_inspect))
    if plugin in _registered_xml_gui_clients:
        return _registered_xml_gui_clients[plugin]

#
# initialization related stuff
#

def pateEventHandler(event):
    def _decorator(func):
        setattr(pate, event, func)
        del func
    return _decorator

def _callAll(plugin, functions, *args, **kwargs):
    if plugin in functions:
        for f in functions[plugin]:
            try:
                f(*args, **kwargs)
            except:
                traceback.print_exc()
                sys.stderr.write('\n')
                # TODO Return smth to a caller, so in case of
                # failed initialization it may report smth to the
                # C++ level and latter can show an error to the user...
                continue

def _simpleEventListener(func):
    # automates the most common decorator pattern: calling a bunch
    # of functions when an event has occurred
    func.functions = dict()
    func.fire = functools.partial(_callAll, functions=func.functions)
    func.clear = func.functions.clear
    return func

def _registerCallback(plugin, event, func):
    if plugin not in event.functions:
        event.functions[plugin] = set()

    event.functions[plugin].add(func)
    return func

@_simpleEventListener
def init(func):
    ''' The function will be called when particular plugin has loaded
        and the configuration has been initiated
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    kDebug('@init: {}/{}'.format(plugin, func.__name__))
    return _registerCallback(plugin, init, func)

@_simpleEventListener
def unload(func):
    ''' The function will be called when particular plugin is being
        unloaded from memory. Clean up any widgets that you have added
        to the interface (toolviews etc).

        ATTENTION Be really careful trying to access any window, view
            or document from the @unload handler: in case of application
            quit everything is dead already!
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    kDebug('@unload: {}/{}'.format(plugin, func.__name__))
    def _module_cleaner():
        kDebug('@unload/cleaner: {}/{}'.format(plugin, func.__name__))
        if plugin in _registered_xml_gui_clients and mainInterfaceWindow() is not None:
            clnt = _registered_xml_gui_clients[plugin]
            clnt.actionCollection().clearAssociatedWidgets()
            clnt.actionCollection().clear()
            mainInterfaceWindow().guiFactory().removeClient(clnt)
            del _registered_xml_gui_clients[plugin]

        if plugin in init.functions:
            kDebug('@unload/init-cleaner: {}/{}'.format(plugin, func.__name__))
            del init.functions[plugin]

        func()

    return _registerCallback(plugin, unload, _module_cleaner)

@_simpleEventListener
def viewCreated(func):
    ''' Calls the function when a new view is created, passing the
        view as a parameter.
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    mainWindow = application.activeMainWindow()
    assert(mainWindow is not None)
    mainWindow.connect(mainWindow, QtCore.SIGNAL('viewCreated(KTextEditor::View*)'), func)
    return func

@_simpleEventListener
def viewChanged(func):
    ''' Calls the function when the view changes.
        To access the new active view, use kate.activeView()

        NOTE The very first call after kate starts the view actually is None!
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    mainWindow = application.activeMainWindow()
    assert(mainWindow is not None)
    mainWindow.connect(mainWindow, QtCore.SIGNAL('viewChanged()'), func)
    return func

def configPage(name, fullName, icon):
    ''' Decorator that adds a configPage into Kate's settings dialog. When the
    item is fired, your function is called.

    Parameters:
        * name -        The text associated with the configPage in the list of
                        config pages.
        * fullName -    The title of the configPage when selected.
        * icon -        An icon to associate with this configPage. Pass a string
                        to use KDE's image loading system or a QPixmap or
                        QIcon to use any custom icon.

    NOTE: Kate may need to be restarted for this decorator to take effect, or
    to remove all traces of the plugin on removal.
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    def _decorator(func):
        a = name, fullName, kdeui.KIcon(icon)
        func.configPage = a
        return func
    return _decorator

def moduleGetConfigPages(module):
    '''Return a list of each module function decorated with @configPage.

    The returned object is [ { function, callable, ( name, fullName, icon ) }... ].
    '''
    result = []
    for k, v in module.__dict__.items():
        if inspect.isfunction(v) and hasattr(v, 'configPage'):
            result.append((k, v, v.configPage))
    return result

#
# Actions related stuff
#

class _HandledException(Exception):
    def __init__(self, message):
        super(_HandledException, self).__init__(message)

class _catchAllHandler(object):
    '''Standard error handling for plugin actions.'''
    def __init__(self, f):
        self.f = f

    def __call__(self):
        try:
            return self.f()
        except _HandledException:
            raise
        except Exception as e:
            txt = ''.join(traceback.format_exception(*sys.exc_info()))
            KMessageBox.error(
                None
              , txt
              , i18nc('@title:window', 'Error in action <icode>%1</icode>', self.f.__name__)
              )
            raise _HandledException(txt)

class InvalidAction(Exception):
    pass

def action(func):
    ''' Decorator that adds an action to the menu bar. When the item is fired,
        your function is called
    '''
    plugin = sys._getframe(1).f_globals['__name__']
    kDebug('@action: {}/{}'.format(plugin, func.__name__))
    ui_file = kdecore.KGlobal.dirs().findResource('appdata', 'pate/{}_ui.rc'.format(plugin))
    if not ui_file:
        ui_file = kdecore.KGlobal.dirs().findResource('appdata', 'pate/{}/{}_ui.rc'.format(plugin, plugin))
        if not ui_file:
            # TODO Report an error: found an action w/o corresponding ui.rc file!
            return func

    # Found UI resource file
    #kDebug('ui_file={}'.format(repr(ui_file)))

    # Get the XML GUI client or create a new one
    clnt = None
    if plugin not in _registered_xml_gui_clients:
        kDebug('@action: make XMLGUIClient for plugin={}'.format(plugin))
        clnt = kdeui.KXMLGUIClient()
        clnt.replaceXMLFile(ui_file,ui_file)
    else:
        clnt = _registered_xml_gui_clients[plugin]
        mainInterfaceWindow().guiFactory().removeClient(clnt)

    # Find an action details
    gui = xml.dom.minidom.parse(ui_file)
    found = False
    for tag in gui.getElementsByTagName('Action'):
        # Action at least must have a name and text
        # and name must be the same as a function name marked w/ @action decorator
        want_it = 'name' in tag.attributes.keys() \
            and 'text' in tag.attributes.keys() \
            and func.__name__ == tag.attributes['name'].value
        if want_it:
            found = True
            # Get mandatory attributes
            name = tag.attributes['name'].value
            text = tag.attributes['text'].value
            act = clnt.actionCollection().addAction(name)
            act.setText(text)
            kDebug('@action/found: {} --> "{}"'.format(name, text))
            # Get optional attributes
            if 'shortcut' in tag.attributes.keys():
                act.setShortcut(QtGui.QKeySequence(tag.attributes['shortcut'].value))
            if 'icon' in tag.attributes.keys():
                act.setIcon(kdeui.KIcon(tag.attributes['icon'].value))
            if 'iconText' in tag.attributes.keys():
                act.setIconText(tag.attributes['iconText'].value)
            if 'toolTip' in tag.attributes.keys():
                act.setToolTip(tag.attributes['toolTip'].value)
            if 'whatsThis' in tag.attributes.keys():
                act.setWhatsThis(tag.attributes['whatsThis'].value)
            # Connect it to the function
            act.connect(act, QtCore.SIGNAL('triggered()'), func)
            # Make an action accessible from the targeted function as its attribute
            setattr(func, 'action', act)
            # No need to scan anything else!!!
            # ATTENTION It is why the very first action declaration
            # MUST contain everything! While others, w/ same name,
            # may have only name! For example, if same action must be
            # added to toolbar or context menu...
            break

    if not found:
        raise InvalidAction(
            i18nc(
                '@info:tooltip'
              , 'Invalid <filename>%1</filename> file: No action with name <icode>%2</icode> defined'
              , ui_file
              , func.__name__
              )
          )

    if plugin not in _registered_xml_gui_clients:
        _registered_xml_gui_clients[plugin] = clnt

    mainInterfaceWindow().guiFactory().addClient(clnt)

    return func
<|endoftext|>


<commit_before># Originally written by Brian Wallace (@botnet_hunter):
# http://blog.cylance.com/a-study-in-bots-blackshades-net

import os
import sys
import string
import re

prng_seed = 0

def is_valid_config(config):
    if config[:3] != "\x0c\x0c\x0c":
        return False
    if config.count("\x0C\x0C\x0C") < 15:
        return False
    return True

def get_next_rng_value():
    global prng_seed
    prng_seed = ((prng_seed * 1140671485 + 12820163) & 0xffffff)
    return prng_seed / 65536

def decrypt_configuration(hex):
    global prng_seed
    ascii = hex.decode('hex')
    tail = ascii[0x20:]

    pre_check = []
    for x in xrange(3):
        pre_check.append(ord(tail[x]) ^ 0x0c)

    for x in xrange(0xffffff):
        prng_seed = x
        if get_next_rng_value() != pre_check[0] or get_next_rng_value() != pre_check[1] or get_next_rng_value() != pre_check[2]:
            continue
        prng_seed = x
        config = "".join((chr(ord(c) ^ int(get_next_rng_value())) for c in tail))
        if is_valid_config(config):
            return config.split("\x0c\x0c\x0c")
    return None

def config_extract(raw_data):
    config_pattern = re.findall('[0-9a-fA-F]{154,}', raw_data)
    for s in config_pattern:
        if (len(s) % 2) == 1:
            s = s[:-1]
        return s
    return None

def config_parser(config):
    config_dict = {}
    config_dict['Domain'] = config[1]
    config_dict['Client Control Port'] = config[2]
    config_dict['Client Transfer Port'] = config[3]
    config_dict['Campaign ID'] = config[4]
    config_dict['File Name'] = config[5]
    config_dict['Install Path'] = config[6]
    config_dict['Registry Key'] = config[7]
    config_dict['ActiveX Key'] = config[8]
    config_dict['Install Flag'] = config[9]
    config_dict['Hide File'] = config[10]
    config_dict['Melt File'] = config[11]
    config_dict['Delay'] = config[12]
    config_dict['USB Spread'] = config[13]
    config_dict['Mutex'] = config[14]
    config_dict['Log File'] = config[15]
    config_dict['Folder Name'] = config[16]
    config_dict['Smart DNS'] = config[17]
    config_dict['Protect Process'] = config[18]
    return config_dict

def config(data):
    raw_config = config_extract(data)
    if raw_config is not None:
        config = decrypt_configuration(raw_config)
        if config is not None and len(config) > 15:
            sorted_config = config_parser(config)
            return sorted_config
    return None

<commit_msg>Small style fixes<commit_after># Originally written by Brian Wallace (@botnet_hunter):
# http://blog.cylance.com/a-study-in-bots-blackshades-net

import os
import sys
import string
import re

PRNG_SEED = 0

def is_valid_config(config):
    if config[:3] != "\x0c\x0c\x0c":
        return False
    if config.count("\x0C\x0C\x0C") < 15:
        return False
    return True

def get_next_rng_value():
    global PRNG_SEED
    PRNG_SEED = ((PRNG_SEED * 1140671485 + 12820163) & 0xffffff)
    return PRNG_SEED / 65536

def decrypt_configuration(hex):
    global PRNG_SEED
    ascii = hex.decode('hex')
    tail = ascii[0x20:]

    pre_check = []
    for x in xrange(3):
        pre_check.append(ord(tail[x]) ^ 0x0c)

    for x in xrange(0xffffff):
        PRNG_SEED = x
        if get_next_rng_value() != pre_check[0] or get_next_rng_value() != pre_check[1] or get_next_rng_value() != pre_check[2]:
            continue
        PRNG_SEED = x
        config = "".join((chr(ord(c) ^ int(get_next_rng_value())) for c in tail))
        if is_valid_config(config):
            return config.split("\x0c\x0c\x0c")

def config_extract(raw_data):
    config_pattern = re.findall('[0-9a-fA-F]{154,}', raw_data)
    for s in config_pattern:
        if (len(s) % 2) == 1:
            s = s[:-1]
        return s

def config_parser(config):
    config_dict = {}
    config_dict['Domain'] = config[1]
    config_dict['Client Control Port'] = config[2]
    config_dict['Client Transfer Port'] = config[3]
    config_dict['Campaign ID'] = config[4]
    config_dict['File Name'] = config[5]
    config_dict['Install Path'] = config[6]
    config_dict['Registry Key'] = config[7]
    config_dict['ActiveX Key'] = config[8]
    config_dict['Install Flag'] = config[9]
    config_dict['Hide File'] = config[10]
    config_dict['Melt File'] = config[11]
    config_dict['Delay'] = config[12]
    config_dict['USB Spread'] = config[13]
    config_dict['Mutex'] = config[14]
    config_dict['Log File'] = config[15]
    config_dict['Folder Name'] = config[16]
    config_dict['Smart DNS'] = config[17]
    config_dict['Protect Process'] = config[18]
    return config_dict

def config(data):
    raw_config = config_extract(data)
    if raw_config:
        config = decrypt_configuration(raw_config)
        if config and len(config) > 15:
            sorted_config = config_parser(config)
            return sorted_config
<|endoftext|>