<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools
import models # damn!
import pprint
import sys
from iatidq import db

COL_INDICATOR = 0
COL_TEST = 1
COL_RESULTS_DATA = 2
COL_RESULTS_NUM = 3
COL_HIERARCHY = 4
COL_PACKAGE = 5

class NoRelevantResults(Exception): pass

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def remove_empty_dicts(h):
    has_keys = lambda kvp: len(kvp[1])
    return dict([
            (K, dict(filter(has_keys, V.items())))
            for K, V in h.items()
            ])

class TestInfo(object):
    def __init__(self):
        self.tests = dict([ (t.id, t) for t in
                            models.Test.query.all() ])

    def as_dict(self, test_id, results_raw_score, results_num, sampling_ok):
        test = self.tests[test_id]

        sampling_factor = {
            True: 1.0,
            False: 0.0
            }[sampling_ok]

        results_pct = results_raw_score * sampling_factor

        return {
            "test": {
                "id": test_id,
                "name": test.name,
                "description": test.description,
                "test_group": test.test_group,
                "test_level": test.test_level
                },
            "results_pct": results_pct,
            "results_num": results_num,
            "sampling_ok": sampling_ok,
            "results_raw_score": results_raw_score
            }

class IndicatorInfo(object):
    def __init__(self):
        self.inds = dict([ (i.id, i.as_dict()) for i in
                           models.Indicator.query.all() ])

    def as_dict(self, indicator_id):
        return self.inds[indicator_id]

    def as_dict_minus_group(self, indicator_id):
        tmp = dict(self.inds[indicator_id]) # deep copy or you mutate >1 time
        del(tmp['indicatorgroup_id'])
        return tmp

def publisher_indicators(indicator_info, indicators, indicators_tests,
                         simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    def per_indicator(indicator):
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        relevant = lambda test: (indicator, test) in indicators_tests

        for test in filter(relevant, simple_out.keys()):
            indic_info = simple_out[test]
            results_pct += indic_info["results_pct"]
            results_num += indic_info["results_num"]
            results_weighted_pct_average_numerator += (
                indic_info["results_pct"] *
                indic_info["results_num"]
                )
            indicator_test_data.append(indic_info)

        return {
            "indicator": indicator_info.as_dict_minus_group(indicator),
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }

    return dict([ (i, per_indicator(i)) for i in indicators ])

def publisher_simple(all_test_info, out, cdtns, indicator_lookup, indicators):
    hierarchies = set(out)
    tests = set()

    for h in hierarchies:
        tests.update(set(out[h]))

    def per_test(t):
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        def relevant(hierarchy):
            if t not in out[hierarchy]:
                return False

            return cdtns.is_relevant(t, hierarchy)

        for hierarchy in filter(relevant, hierarchies):
            test_info = out[hierarchy][t]

            results_pct += test_info["results_pct"]
            results_num += test_info["results_num"]
            results_weighted_pct_average_numerator += (
                test_info["results_pct"] *
                test_info["results_num"]
                )

        if results_num == 0:
            raise NoRelevantResults("Results_num == 0 for test: %d" % t)

        tmp = all_test_info.as_dict(
            t,
            (results_weighted_pct_average_numerator / results_num),
            results_num,
            True
            )
        indicator_id = indicator_lookup[t]
        tmp["indicator"] = indicators.as_dict(indicator_id)
        return tmp

    return dict([ (t, per_test(t)) for t in tests ])

class Summary(object):
    def __init__(self, data, conditions):
        self.data = data
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()
        self._summary = self.calculate()

    def summary(self):
        return self._summary

    def generate_summaries(self, hierarchies, tests, summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def summarise_results(self, hierarchies,
                      tests, indicators,
                      indicators_tests, indicator_lookup,
                      summary_f):

        def add_condition(i):
            h, t, tdata = i
            if self.conditions.has_condition(t, h):
                tdata["condition"] = self.conditions.get_condition(t, h)
            return h, t, tdata

        summaries = (add_condition(i) for i in self.generate_summaries(
                hierarchies, tests, summary_f))

        tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])

        out = reform_dict(tmp_out)

        out = remove_empty_dicts(out)
        return self.add_indicator_info(out, indicators,
                                       indicators_tests, indicator_lookup)

    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):
        return out

class PublisherSummary(Summary):
    pass

class NewPublisherSummary(PublisherSummary):
    def __init__(self, conditions, organisation_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()

        where_clause = '''WHERE organisation_id = %d AND
                            aggregateresulttype_id = %d''' % (organisation_id,
                                                              aggregation_type)

        self._summary = self.calculate(where_clause)

    def calculate(self, where_clause):
        # make list of data; hand over to
        # summarise_results(hierarchies, tests, indicators,
        #                   indicators_tests, summary_f):
        conn = db.session.connection()

        sql = '''SELECT DISTINCT result_hierarchy
                   FROM aggregateresult
                   %s;'''
        stmt = sql % where_clause
        hierarchies = [ h[0] for h in conn.execute(stmt) ]

        sql = '''SELECT DISTINCT indicator_id, test_id
                   FROM aggregateresult
                   JOIN indicatortest USING (test_id)
                   %s;'''
        stmt = sql % where_clause
        indicators_tests = [ it for it in conn.execute(stmt) ]
        tests = [ it[1] for it in indicators_tests ]
        indicators = [ it[0] for it in indicators_tests ]

        indicator_lookup = dict([ (it[1], it[0]) for it in indicators_tests ])

        sql = '''SELECT result_hierarchy, test_id, AVG(results_data) AS pct,
                        SUM(results_num) AS total_activities
                   FROM aggregateresult
                   %s
                   GROUP BY test_id, result_hierarchy;'''
        stmt = sql % where_clause
        data = dict([ ((ar[0], ar[1]), ar) for ar in (conn.execute(stmt)) ])
        conn.close()
        del(conn)

        def summary_f(hierarchy, test_id):
            key = (hierarchy, test_id)
            if key not in data:
                return {}

            aresult = data[key]
            sampling_ok = True
            indicator_id = indicator_lookup[test_id]

            tmp = self.tests.as_dict(test_id, aresult[2], aresult[3],
                                     sampling_ok)
            tmp["indicator"] = self.indicators.as_dict(indicator_id)
            return tmp

        return self.summarise_results(hierarchies, tests, indicators,
                                      indicators_tests, indicator_lookup,
                                      summary_f)

class NewPackageSummary(NewPublisherSummary):
    def __init__(self, conditions, package_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()

        where_clause = '''WHERE package_id = %d AND
                            aggregateresulttype_id = %d''' % (package_id,
                                                              aggregation_type)

        self._summary = self.calculate(where_clause)

class PublisherIndicatorsSummary(NewPublisherSummary):
    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):

        simple_out = publisher_simple(self.tests, out, self.conditions,
                                      indicator_lookup, self.indicators)
        return publisher_indicators(self.indicators, indicators,
                                    indicators_tests, simple_out)

from models import *

class OrgConditions(object):
    def __init__(self, organisation_id):
        # None is passed as organisation_id for case of no conditions wanted
        if organisation_id is None:
            self._conditions = {}
            return

        cc = OrganisationCondition.query.filter_by(
            organisation_id=organisation_id
            ).all()
        self._conditions = dict(map(lambda x: (
                    (x.test_id, x.condition, x.condition_value),
                    (x.operation, x.description)
                    ), cc))

    def _key(self, test_id, hierarchy):
        return (test_id, 'activity hierarchy', str(hierarchy))

    def is_relevant(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)

        if not self._conditions:
            return True

        if key not in self._conditions:
            return True

        if self._conditions[key][0] == 0:
            return False

        return True

    def has_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return key in self._conditions

    def get_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return self._conditions[key]

class SummaryCreator(object):
    @property
    def summary(self):
        return self._summary

    @property
    def aggregate_results(self):
        return self._aggregate_results

class PublisherSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = NewPublisherSummary(pconditions,
                                            organisation_id,
                                            aggregation_type)

class PackageSummaryCreator(SummaryCreator):
    def __init__(self, package_id, latest_runtime, aggregation_type):

        self._summary = NewPackageSummary(OrgConditions(None),
                                          package_id,
                                          aggregation_type).summary()

class PublisherIndicatorsSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = PublisherIndicatorsSummary(pconditions,
                                                   organisation_id,
                                                   aggregation_type)

<commit_msg>break out sampling<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools
import models # damn!
import pprint
import sys
from iatidq import db

COL_INDICATOR = 0
COL_TEST = 1
COL_RESULTS_DATA = 2
COL_RESULTS_NUM = 3
COL_HIERARCHY = 4
COL_PACKAGE = 5

class NoRelevantResults(Exception): pass

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def remove_empty_dicts(h):
    has_keys = lambda kvp: len(kvp[1])
    return dict([
            (K, dict(filter(has_keys, V.items())))
            for K, V in h.items()
            ])

class TestInfo(object):
    def __init__(self):
        self.tests = dict([ (t.id, t) for t in
                            models.Test.query.all() ])

    def as_dict(self, test_id, results_raw_score, results_num, sampling_ok):
        test = self.tests[test_id]

        sampling_factor = {
            True: 1.0,
            False: 0.0
            }[sampling_ok]

        results_pct = results_raw_score * sampling_factor

        return {
            "test": {
                "id": test_id,
                "name": test.name,
                "description": test.description,
                "test_group": test.test_group,
                "test_level": test.test_level
                },
            "results_pct": results_pct,
            "results_num": results_num,
            "sampling_ok": sampling_ok,
            "results_raw_score": results_raw_score
            }

class IndicatorInfo(object):
    def __init__(self):
        self.inds = dict([ (i.id, i.as_dict()) for i in
                           models.Indicator.query.all() ])

    def as_dict(self, indicator_id):
        return self.inds[indicator_id]

    def as_dict_minus_group(self, indicator_id):
        tmp = dict(self.inds[indicator_id]) # deep copy or you mutate >1 time
        del(tmp['indicatorgroup_id'])
        return tmp

def publisher_indicators(indicator_info, indicators, indicators_tests,
                         simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    def per_indicator(indicator):
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        relevant = lambda test: (indicator, test) in indicators_tests

        for test in filter(relevant, simple_out.keys()):
            indic_info = simple_out[test]
            results_pct += indic_info["results_pct"]
            results_num += indic_info["results_num"]
            results_weighted_pct_average_numerator += (
                indic_info["results_pct"] *
                indic_info["results_num"]
                )
            indicator_test_data.append(indic_info)

        return {
            "indicator": indicator_info.as_dict_minus_group(indicator),
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }

    return dict([ (i, per_indicator(i)) for i in indicators ])

def publisher_simple(all_test_info, out, cdtns, indicator_lookup, indicators):
    hierarchies = set(out)
    tests = set()

    for h in hierarchies:
        tests.update(set(out[h]))

    def per_test(t):
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        def relevant(hierarchy):
            if t not in out[hierarchy]:
                return False

            return cdtns.is_relevant(t, hierarchy)

        for hierarchy in filter(relevant, hierarchies):
            test_info = out[hierarchy][t]

            results_pct += test_info["results_pct"]
            results_num += test_info["results_num"]
            results_weighted_pct_average_numerator += (
                test_info["results_pct"] *
                test_info["results_num"]
                )

        if results_num == 0:
            raise NoRelevantResults("Results_num == 0 for test: %d" % t)

        sampling_ok = True

        tmp = all_test_info.as_dict(
            t,
            (results_weighted_pct_average_numerator / results_num),
            results_num,
            sampling_ok
            )
        indicator_id = indicator_lookup[t]
        tmp["indicator"] = indicators.as_dict(indicator_id)
        return tmp

    return dict([ (t, per_test(t)) for t in tests ])

class Summary(object):
    def __init__(self, data, conditions):
        self.data = data
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()
        self._summary = self.calculate()

    def summary(self):
        return self._summary

    def generate_summaries(self, hierarchies, tests, summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def summarise_results(self, hierarchies,
                      tests, indicators,
                      indicators_tests, indicator_lookup,
                      summary_f):

        def add_condition(i):
            h, t, tdata = i
            if self.conditions.has_condition(t, h):
                tdata["condition"] = self.conditions.get_condition(t, h)
            return h, t, tdata

        summaries = (add_condition(i) for i in self.generate_summaries(
                hierarchies, tests, summary_f))

        tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])

        out = reform_dict(tmp_out)

        out = remove_empty_dicts(out)
        return self.add_indicator_info(out, indicators,
                                       indicators_tests, indicator_lookup)

    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):
        return out

class PublisherSummary(Summary):
    pass

class NewPublisherSummary(PublisherSummary):
    def __init__(self, conditions, organisation_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()

        where_clause = '''WHERE organisation_id = %d AND
                            aggregateresulttype_id = %d''' % (organisation_id,
                                                              aggregation_type)

        self._summary = self.calculate(where_clause)

    def calculate(self, where_clause):
        # make list of data; hand over to
        # summarise_results(hierarchies, tests, indicators,
        #                   indicators_tests, summary_f):
        conn = db.session.connection()

        sql = '''SELECT DISTINCT result_hierarchy
                   FROM aggregateresult
                   %s;'''
        stmt = sql % where_clause
        hierarchies = [ h[0] for h in conn.execute(stmt) ]

        sql = '''SELECT DISTINCT indicator_id, test_id
                   FROM aggregateresult
                   JOIN indicatortest USING (test_id)
                   %s;'''
        stmt = sql % where_clause
        indicators_tests = [ it for it in conn.execute(stmt) ]
        tests = [ it[1] for it in indicators_tests ]
        indicators = [ it[0] for it in indicators_tests ]

        indicator_lookup = dict([ (it[1], it[0]) for it in indicators_tests ])

        sql = '''SELECT result_hierarchy, test_id, AVG(results_data) AS pct,
                        SUM(results_num) AS total_activities
                   FROM aggregateresult
                   %s
                   GROUP BY test_id, result_hierarchy;'''
        stmt = sql % where_clause
        data = dict([ ((ar[0], ar[1]), ar) for ar in (conn.execute(stmt)) ])
        conn.close()
        del(conn)

        def summary_f(hierarchy, test_id):
            key = (hierarchy, test_id)
            if key not in data:
                return {}

            aresult = data[key]
            sampling_ok = True
            indicator_id = indicator_lookup[test_id]

            tmp = self.tests.as_dict(test_id, aresult[2], aresult[3],
                                     sampling_ok)
            tmp["indicator"] = self.indicators.as_dict(indicator_id)
            return tmp

        return self.summarise_results(hierarchies, tests, indicators,
                                      indicators_tests, indicator_lookup,
                                      summary_f)

class NewPackageSummary(NewPublisherSummary):
    def __init__(self, conditions, package_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()

        where_clause = '''WHERE package_id = %d AND
                            aggregateresulttype_id = %d''' % (package_id,
                                                              aggregation_type)

        self._summary = self.calculate(where_clause)

class PublisherIndicatorsSummary(NewPublisherSummary):
    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):

        simple_out = publisher_simple(self.tests, out, self.conditions,
                                      indicator_lookup, self.indicators)
        return publisher_indicators(self.indicators, indicators,
                                    indicators_tests, simple_out)

from models import *

class OrgConditions(object):
    def __init__(self, organisation_id):
        # None is passed as organisation_id for case of no conditions wanted
        if organisation_id is None:
            self._conditions = {}
            return

        cc = OrganisationCondition.query.filter_by(
            organisation_id=organisation_id
            ).all()
        self._conditions = dict(map(lambda x: (
                    (x.test_id, x.condition, x.condition_value),
                    (x.operation, x.description)
                    ), cc))

    def _key(self, test_id, hierarchy):
        return (test_id, 'activity hierarchy', str(hierarchy))

    def is_relevant(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)

        if not self._conditions:
            return True

        if key not in self._conditions:
            return True

        if self._conditions[key][0] == 0:
            return False

        return True

    def has_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return key in self._conditions

    def get_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return self._conditions[key]

class SummaryCreator(object):
    @property
    def summary(self):
        return self._summary

    @property
    def aggregate_results(self):
        return self._aggregate_results

class PublisherSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = NewPublisherSummary(pconditions,
                                            organisation_id,
                                            aggregation_type)

class PackageSummaryCreator(SummaryCreator):
    def __init__(self, package_id, latest_runtime, aggregation_type):

        self._summary = NewPackageSummary(OrgConditions(None),
                                          package_id,
                                          aggregation_type).summary()

class PublisherIndicatorsSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = PublisherIndicatorsSummary(pconditions,
                                                   organisation_id,
                                                   aggregation_type)

<|endoftext|>


<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def publisher_mode(mode):
    if mode in ["publisher", "publisher_indicators"]:
        return True
    return False

def publisher_indicators(indicators, indicators_tests, simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    indicators_out = {}
    for indicator, indicatordata in indicators:
        indicators_out[indicator] = {}
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for test, testdata in simple_out.items():
            try:
                testing = (indicator, test)
                if (testing in indicators_tests):
                    results_pct+= simple_out[test]["results_pct"]
                    results_num+= simple_out[test]["results_num"]
                    results_weighted_pct_average_numerator += (simple_out[test]["results_pct"]*simple_out[test]["results_num"])
                    oktest = test
                    indicator_test_data.append(simple_out[test])
            except KeyError:
                pass
        indicators_out[indicator] = {
            "indicator": {
                "id": indicator,
                "name": indicatordata[0],
                "description": indicatordata[1],
                "indicator_type": indicatordata[2],
                "indicator_category_name": indicatordata[3],
                "indicator_subcategory_name": indicatordata[4],
                "longdescription": indicatordata[5],
                "indicator_noformat": indicatordata[6],
                "indicator_ordinal": indicatordata[7],
                "indicator_order": indicatordata[8],
                "indicator_weight": indicatordata[9]
                },
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }
    return indicators_out
    #for test, testdata in simple_out.items():
    #    indicators_out[testdata["indicator"]["id"]] = testdata
    #return indicators_out

def make_summary(test_id, test_name, test_description, test_group,
                 test_level, results_pct, results_num):
    return {
        "test": {
            "id": test_id,
            "name": test_name,
            "description": test_description,
            "test_group": test_group,
            "test_level": test_level
            },
        "results_pct": results_pct,
        "results_num": results_num
        }

def publisher_simple(out, cdtns):
    simple_out = {}
    hierarchies = set(out)
    tests = set()
    for h in hierarchies:
        tests.update(set(out[h]))
    for t in tests:
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for hierarchy in hierarchies:
            try:
                key = (t,'activity hierarchy', str(hierarchy))
                try:
                    if ((cdtns) and (key in cdtns) and (cdtns[key][0]==0)):
                        continue
                except KeyError:
                    pass
                results_pct+= out[hierarchy][t]["results_pct"]
                results_num+= out[hierarchy][t]["results_num"]
                results_weighted_pct_average_numerator += (out[hierarchy][t]["results_pct"]*out[hierarchy][t]["results_num"])
                okhierarchy = hierarchy
            except KeyError:
                pass

        tmp = make_summary(
            out[okhierarchy][t]['test']["id"],
            out[okhierarchy][t]['test']["name"],
            out[okhierarchy][t]['test']["description"],
            out[okhierarchy][t]['test']["test_group"],
            out[okhierarchy][t]['test']["test_level"],
            (results_weighted_pct_average_numerator/results_num),
            results_num
            )
        tmp["indicator"] = out[okhierarchy][t]['indicator']
        simple_out[t] = tmp
    return simple_out

def sum_for_publishers(packages, d, h, t):
    # aggregate data across multiple packages for a single publisher

    # for each package, add percentage for each
    total_pct = 0
    total_activities = 0

    # need below to only include packages that are in this hierarchy
    packages_in_hierarchy = 0

    relevant = lambda p: (h, t, p) in d
    relevant_data = map(lambda p: d[(h, t, p)], filter(relevant, packages))

    pct = lambda i: i[2]
    activities = lambda i: i[3]

    try:
        total_pct = reduce(operator.add, map(pct, relevant_data))
        total_activities = reduce(operator.add, map(activities, relevant_data))
    except Exception:
        total_pct = 0.0
        total_activities = 0
    packages_in_hierarchy = len(relevant_data)

    if total_activities <= 0:
        return {}

    ok_tdata = relevant_data[-1] ## FIXME: this is obviously wrong

    tmp = make_summary(
        ok_tdata[1].id,
        ok_tdata[1].name,
        ok_tdata[1].description,
        ok_tdata[1].test_group,
        ok_tdata[1].test_level,
        int(float(total_pct/packages_in_hierarchy)),
        total_activities
        )
    tmp["indicator"] = ok_tdata[0]
    tmp["result_hierarchy"] = total_activities
    return tmp

def sum_default(d, h, t):
    # return data for this single package
    data = d.get((h, t), None)
    if data is None:
        return None

    tmp = make_summary(
        data[1].id,
        data[1].name,
        data[1].description,
        data[1].test_group,
        data[1].test_level,
        data[2],
        data[3])
    tmp["result_hierarchy"] = data[4]
    return tmp

def summarise_results(conditions, mode, hierarchies,
                      tests, cdtns, indicators,
                      indicators_tests, packages, d, summary):

    def summaries(summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def add_condition(i):
        h, t, tdata = i
        if conditions:
            key = (t, 'activity hierarchy', str(h))
            if key in cdtns:
                tdata["condition"] = cdtns[key]
        return h, t, tdata

    summaries = (add_condition(i) for i in summaries(summary))

    ## FIXME: the bit which removes empty data seems to have been deleted

    tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])
    out = reform_dict(tmp_out)

    if mode != "publisher_indicators":
        return out

    simple_out = publisher_simple(out, cdtns)
    return publisher_indicators(indicators, indicators_tests, simple_out)

def _agr_results(data, conditions, mode):
    """
        data variable looks like this:
            models.Indicator,
            models.Test,
            models.AggregateResult.results_data, ## percentage
            models.AggregateResult.results_num, ## total values
            models.AggregateResult.result_hierarchy

        conditions variable looks like this:
            models.PublisherCondition.query.filter_by(publisher_id=p[1].id).all()

            ======================================
            id = Column(Integer, primary_key=True)
            publisher_id = Column(Integer, ForeignKey('packagegroup.id'))
            test_id = Column(Integer, ForeignKey('test.id'))
            operation = Column(Integer) # show (1) or don't show (0) result
            condition = Column(UnicodeText) # activity level, hierarchy 2
            condition_value = Column(UnicodeText) # True, 2, etc.
            description = Column(UnicodeText)
            file = Column(UnicodeText)
            line = Column(Integer)
            active = Column(Boolean)
            ======================================
        mode can be:
            None = package-specific
            "publisher" = aggregate results for a whole publisher

        TODO:
            in publisher mode, allow weighting of data by package rather than number of activities.
    """

    def gen_hierarchies():
        for i in set(map(lambda x: (x[4]), data)):
            yield i
    hierarchies = gen_hierarchies()

    def gen_tests():
        for i in set(map(lambda x: (x[1].id), data)):
            yield i
    tests = gen_tests()

    cdtns = None
    if conditions:
        cdtns = dict(map(lambda x: (
                    (x.test_id, x.condition, x.condition_value),
                    (x.operation, x.description)
                    ), conditions))

    def setmap(lam):
        return set(map(lam, data))

    def dictmap(lam):
        return dict(map(lam, data))

    if publisher_mode(mode):
        ind_f = lambda x: (
            x[0]["id"], (
                x[0]["name"],
                x[0]["description"],
                x[0]["indicator_type"],
                x[0]["indicator_category_name"],
                x[0]["indicator_subcategory_name"],
                x[0]["longdescription"],
                x[0]["indicator_noformat"],
                x[0]["indicator_ordinal"],
                x[0]["indicator_order"],
                x[0]["indicator_weight"]
                )
            )
        indicators = setmap(ind_f)

        ind_test_f = lambda x: (x[0]["id"], x[1].id)
        indicators_tests = list(setmap(ind_test_f))

        pkg_f = lambda x: x[5]
        packages = setmap(pkg_f)

        d_f = lambda x: ((x[4], x[1].id, x[5]),(x))
        d = dictmap(d_f)

        summary = lambda h, t: sum_for_publishers(packages, d, h, t)

    else:
        d_f = lambda x: ((x[4], x[1].id),(x))
        d = dictmap(d_f)

        summary = lambda h, t: sum_default(d, h, t)

    return summarise_results(conditions, mode, hierarchies,
                             tests, cdtns, indicators,
                             indicators_tests, packages, d, summary)

class Summary(object):
    def __init__(self, data, conditions=None, manual=False):
        self.data = data
        self.conditions = conditions
        self.manual = manual
        self._summary = self.calculate()

    def calculate(self):
        mode = self.get_mode()

        if self.manual:
            return _agr_results(self.data, self.conditions, mode)

        def replace_first(tupl, newval):
            return tuple([newval] + list(tupl)[1:])
        switch_first = lambda t: replace_first(t, t[0].as_dict())
        fixed_data = map(switch_first, self.data)
        return _agr_results(fixed_data, self.conditions, mode)

    def summary(self):
        return self._summary

    def get_mode(self):
        raise

class PublisherSummary(Summary):
    def get_mode(self):
        return "publisher"

class VanillaSummary(Summary):
    def get_mode(self):
        return None

class PublisherIndicatorsSummary(Summary):
    def get_mode(self):
        return "publisher_indicators"
<commit_msg>use classes more<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def publisher_mode(mode):
    if mode in ["publisher", "publisher_indicators"]:
        return True
    return False

def publisher_indicators(indicators, indicators_tests, simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    indicators_out = {}
    for indicator, indicatordata in indicators:
        indicators_out[indicator] = {}
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for test, testdata in simple_out.items():
            try:
                testing = (indicator, test)
                if (testing in indicators_tests):
                    results_pct+= simple_out[test]["results_pct"]
                    results_num+= simple_out[test]["results_num"]
                    results_weighted_pct_average_numerator += (simple_out[test]["results_pct"]*simple_out[test]["results_num"])
                    oktest = test
                    indicator_test_data.append(simple_out[test])
            except KeyError:
                pass
        indicators_out[indicator] = {
            "indicator": {
                "id": indicator,
                "name": indicatordata[0],
                "description": indicatordata[1],
                "indicator_type": indicatordata[2],
                "indicator_category_name": indicatordata[3],
                "indicator_subcategory_name": indicatordata[4],
                "longdescription": indicatordata[5],
                "indicator_noformat": indicatordata[6],
                "indicator_ordinal": indicatordata[7],
                "indicator_order": indicatordata[8],
                "indicator_weight": indicatordata[9]
                },
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }
    return indicators_out
    #for test, testdata in simple_out.items():
    #    indicators_out[testdata["indicator"]["id"]] = testdata
    #return indicators_out

def make_summary(test_id, test_name, test_description, test_group,
                 test_level, results_pct, results_num):
    return {
        "test": {
            "id": test_id,
            "name": test_name,
            "description": test_description,
            "test_group": test_group,
            "test_level": test_level
            },
        "results_pct": results_pct,
        "results_num": results_num
        }

def publisher_simple(out, cdtns):
    simple_out = {}
    hierarchies = set(out)
    tests = set()
    for h in hierarchies:
        tests.update(set(out[h]))
    for t in tests:
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for hierarchy in hierarchies:
            try:
                key = (t,'activity hierarchy', str(hierarchy))
                try:
                    if ((cdtns) and (key in cdtns) and (cdtns[key][0]==0)):
                        continue
                except KeyError:
                    pass
                results_pct+= out[hierarchy][t]["results_pct"]
                results_num+= out[hierarchy][t]["results_num"]
                results_weighted_pct_average_numerator += (out[hierarchy][t]["results_pct"]*out[hierarchy][t]["results_num"])
                okhierarchy = hierarchy
            except KeyError:
                pass

        tmp = make_summary(
            out[okhierarchy][t]['test']["id"],
            out[okhierarchy][t]['test']["name"],
            out[okhierarchy][t]['test']["description"],
            out[okhierarchy][t]['test']["test_group"],
            out[okhierarchy][t]['test']["test_level"],
            (results_weighted_pct_average_numerator/results_num),
            results_num
            )
        tmp["indicator"] = out[okhierarchy][t]['indicator']
        simple_out[t] = tmp
    return simple_out

def sum_for_publishers(packages, d, h, t):
    # aggregate data across multiple packages for a single publisher

    # for each package, add percentage for each
    total_pct = 0
    total_activities = 0

    # need below to only include packages that are in this hierarchy
    packages_in_hierarchy = 0

    relevant = lambda p: (h, t, p) in d
    relevant_data = map(lambda p: d[(h, t, p)], filter(relevant, packages))

    pct = lambda i: i[2]
    activities = lambda i: i[3]

    try:
        total_pct = reduce(operator.add, map(pct, relevant_data))
        total_activities = reduce(operator.add, map(activities, relevant_data))
    except Exception:
        total_pct = 0.0
        total_activities = 0
    packages_in_hierarchy = len(relevant_data)

    if total_activities <= 0:
        return {}

    ok_tdata = relevant_data[-1] ## FIXME: this is obviously wrong

    tmp = make_summary(
        ok_tdata[1].id,
        ok_tdata[1].name,
        ok_tdata[1].description,
        ok_tdata[1].test_group,
        ok_tdata[1].test_level,
        int(float(total_pct/packages_in_hierarchy)),
        total_activities
        )
    tmp["indicator"] = ok_tdata[0]
    tmp["result_hierarchy"] = total_activities
    return tmp

def sum_default(d, h, t):
    # return data for this single package
    data = d.get((h, t), None)
    if data is None:
        return None

    tmp = make_summary(
        data[1].id,
        data[1].name,
        data[1].description,
        data[1].test_group,
        data[1].test_level,
        data[2],
        data[3])
    tmp["result_hierarchy"] = data[4]
    return tmp

def summarise_results(conditions, mode, hierarchies,
                      tests, cdtns, indicators,
                      indicators_tests, packages, d, summary):

    def summaries(summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def add_condition(i):
        h, t, tdata = i
        if conditions:
            key = (t, 'activity hierarchy', str(h))
            if key in cdtns:
                tdata["condition"] = cdtns[key]
        return h, t, tdata

    summaries = (add_condition(i) for i in summaries(summary))

    ## FIXME: the bit which removes empty data seems to have been deleted

    tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])
    out = reform_dict(tmp_out)

    if mode != "publisher_indicators":
        return out

    simple_out = publisher_simple(out, cdtns)
    return publisher_indicators(indicators, indicators_tests, simple_out)

class Summary(object):
    def __init__(self, data, conditions=None, manual=False):
        self.data = data
        self.conditions = conditions
        self.manual = manual
        self._summary = self.calculate()

    def calculate(self):
        mode = self.get_mode()

        if self.manual:
            return self.aggregate(self.data)

        def replace_first(tupl, newval):
            return tuple([newval] + list(tupl)[1:])
        switch_first = lambda t: replace_first(t, t[0].as_dict())
        fixed_data = map(switch_first, self.data)
        return self.aggregate(fixed_data)

    def summary(self):
        return self._summary

    def get_mode(self):
        raise

    def aggregate(self, data):
        def gen_hierarchies():
            for i in set(map(lambda x: (x[4]), data)):
                yield i
        hierarchies = gen_hierarchies()

        def gen_tests():
            for i in set(map(lambda x: (x[1].id), data)):
                yield i
        tests = gen_tests()

        cdtns = None
        if self.conditions:
            cdtns = dict(map(lambda x: (
                        (x.test_id, x.condition, x.condition_value),
                        (x.operation, x.description)
                        ), self.conditions))

        def setmap(lam):
            return set(map(lam, data))

        def dictmap(lam):
            return dict(map(lam, data))

        if publisher_mode(self.get_mode()):
            ind_f = lambda x: (
                x[0]["id"], (
                    x[0]["name"],
                    x[0]["description"],
                    x[0]["indicator_type"],
                    x[0]["indicator_category_name"],
                    x[0]["indicator_subcategory_name"],
                    x[0]["longdescription"],
                    x[0]["indicator_noformat"],
                    x[0]["indicator_ordinal"],
                    x[0]["indicator_order"],
                    x[0]["indicator_weight"]
                    )
                )
            indicators = setmap(ind_f)

            ind_test_f = lambda x: (x[0]["id"], x[1].id)
            indicators_tests = list(setmap(ind_test_f))

            pkg_f = lambda x: x[5]
            packages = setmap(pkg_f)

            d_f = lambda x: ((x[4], x[1].id, x[5]),(x))
            d = dictmap(d_f)

            summary = lambda h, t: sum_for_publishers(packages, d, h, t)

        else:
            d_f = lambda x: ((x[4], x[1].id),(x))
            d = dictmap(d_f)

            summary = lambda h, t: sum_default(d, h, t)

        return summarise_results(self.conditions, self.get_mode(), hierarchies,
                                 tests, cdtns, indicators,
                                 indicators_tests, packages, d, summary)

class PublisherSummary(Summary):
    def get_mode(self):
        return "publisher"

class VanillaSummary(Summary):
    def get_mode(self):
        return None

class PublisherIndicatorsSummary(Summary):
    def get_mode(self):
        return "publisher_indicators"
<|endoftext|>


<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def publisher_mode(mode):
    if mode in ["publisher", "publisher_simple", "publisher_indicators"]:
        return True
    return False

def publisher_indicators(indicators, indicators_tests, simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    indicators_out = {}
    for indicator, indicatordata in indicators:
        indicators_out[indicator] = {}
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for test, testdata in simple_out.items():
            try:
                testing = (indicator, test)
                if (testing in indicators_tests):
                    results_pct+= simple_out[test]["results_pct"]
                    results_num+= simple_out[test]["results_num"]
                    results_weighted_pct_average_numerator += (simple_out[test]["results_pct"]*simple_out[test]["results_num"])
                    oktest = test
                    indicator_test_data.append(simple_out[test])
            except KeyError:
                pass
        indicators_out[indicator] = {
            "indicator": {
                "id": indicator,
                "name": indicatordata[0],
                "description": indicatordata[1],
                "indicator_type": indicatordata[2],
                "indicator_category_name": indicatordata[3],
                "indicator_subcategory_name": indicatordata[4]
                },
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }
    return indicators_out
    #for test, testdata in simple_out.items():
    #    indicators_out[testdata["indicator"]["id"]] = testdata
    #return indicators_out

def make_summary(test_id, test_description, test_group,
                 results_pct, results_num):
    return {
        "test": {
            "id": test_id,
            "description": test_description,
            "test_group": test_group
            },
        "results_pct": results_pct,
        "results_num": results_num
        }

def publisher_simple(out, cdtns):
    simple_out = {}
    hierarchies = set(out)
    tests = set()
    for h in hierarchies:
        tests.update(set(out[h]))
    for t in tests:
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for hierarchy in hierarchies:
            try:
                key = (t,'activity hierarchy', str(hierarchy))
                try:
                    if ((cdtns) and (key in cdtns) and (cdtns[key][0]==0)):
                        continue
                except KeyError:
                    pass
                results_pct+= out[hierarchy][t]["results_pct"]
                results_num+= out[hierarchy][t]["results_num"]
                results_weighted_pct_average_numerator += (out[hierarchy][t]["results_pct"]*out[hierarchy][t]["results_num"])
                okhierarchy = hierarchy
            except KeyError:
                pass

        tmp = make_summary(
            out[okhierarchy][t]['test']["id"],
            out[okhierarchy][t]['test']["description"],
            out[okhierarchy][t]['test']["test_group"],
            (results_weighted_pct_average_numerator/results_num),
            results_num
            )
        tmp["indicator"] = out[okhierarchy][t]['indicator']
        simple_out[t] = tmp
    return simple_out

def sum_for_publishers(packages, d, h, t):
    # aggregate data across multiple packages for a single publisher

    # for each package, add percentage for each
    total_pct = 0
    total_activities = 0

    # need below to only include packages that are in this hierarchy
    packages_in_hierarchy = 0

    relevant = lambda p: (h, t, p) in d
    relevant_data = map(lambda p: d[(h, t, p)], filter(relevant, packages))

    pct = lambda i: i[2]
    activities = lambda i: i[3]

    try:
        total_pct = reduce(operator.add, map(pct, relevant_data))
        total_activities = reduce(operator.add, map(activities, relevant_data))
    except Exception:
        total_pct = 0.0
        total_activities = 0
    packages_in_hierarchy = len(relevant_data)

    if total_activities <= 0:
        return {}

    ok_tdata = relevant_data[-1] ## FIXME: this is obviously wrong

    tmp = make_summary(
        ok_tdata[1].id,
        ok_tdata[1].description,
        ok_tdata[1].test_group,
        int(float(total_pct/packages_in_hierarchy)),
        total_activities
        )
    tmp["indicator"] = ok_tdata[0]
    tmp["result_hierarchy"] = total_activities
    return tmp

def sum_default(d, h, t):
    # return data for this single package
    data = d.get((h, t), None)
    if data is None:
        return None

    tmp = make_summary(
        data[1].id,
        data[1].description,
        data[1].test_group,
        data[2],
        data[3])
    tmp["result_hierarchy"] = data[4]
    return tmp

def summarise_results(conditions, mode, hierarchies,
                      tests, cdtns, indicators,
                      indicators_tests, packages, d, summary):

    def summaries(summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def add_condition(i):
        h, t, tdata = i
        if conditions:
            key = (t, 'activity hierarchy', str(h))
            if key in cdtns:
                tdata["condition"] = cdtns[key]
        return h, t, tdata

    summaries = (add_condition(i) for i in summaries(summary))

    ## FIXME: the bit which removes empty data seems to have been deleted

    tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])
    out = reform_dict(tmp_out)

    if mode not in ["publisher_simple", "publisher_indicators"]:
        return out

    simple_out = publisher_simple(out, cdtns)

    if mode != "publisher_indicators":
        return simple_out

    return publisher_indicators(indicators, indicators_tests, simple_out)

def _agr_results(data, conditions=None, mode=None):
    """
        data variable looks like this:
            models.Indicator,
            models.Test,
            models.AggregateResult.results_data, ## percentage
            models.AggregateResult.results_num, ## total values
            models.AggregateResult.result_hierarchy

        conditions variable looks like this:
            models.PublisherCondition.query.filter_by(publisher_id=p[1].id).all()

            ======================================
            id = Column(Integer, primary_key=True)
            publisher_id = Column(Integer, ForeignKey('packagegroup.id'))
            test_id = Column(Integer, ForeignKey('test.id'))
            operation = Column(Integer) # show (1) or don't show (0) result
            condition = Column(UnicodeText) # activity level, hierarchy 2
            condition_value = Column(UnicodeText) # True, 2, etc.
            description = Column(UnicodeText)
            file = Column(UnicodeText)
            line = Column(Integer)
            active = Column(Boolean)
            ======================================
        mode can be:
            None = package-specific
            "publisher" = aggregate results for a whole publisher

        TODO:
            in publisher mode, allow weighting of data by package rather than number of activities.
    """

    def gen_hierarchies():
        for i in set(map(lambda x: (x[4]), data)):
            yield i
    hierarchies = gen_hierarchies()

    def gen_tests():
        for i in set(map(lambda x: (x[1].id), data)):
            yield i
    tests = gen_tests()

    cdtns = None
    if conditions:
        cdtns = dict(map(lambda x: (
                    (x.test_id, x.condition, x.condition_value),
                    (x.operation, x.description)
                    ), conditions))

    def setmap(lam):
        return set(map(lam, data))

    def dictmap(lam):
        return dict(map(lam, data))

    if publisher_mode(mode):
        ind_f = lambda x: (x[0]["id"], (x[0]["name"], x[0]["description"], x[0]["indicator_type"], x[0]["indicator_category_name"], x[0]["indicator_subcategory_name"]))
        indicators = setmap(ind_f)

        ind_test_f = lambda x: (x[0]["id"], x[1].id)
        indicators_tests = list(setmap(ind_test_f))

        pkg_f = lambda x: x[5]
        packages = setmap(pkg_f)

        d_f = lambda x: ((x[4], x[1].id, x[5]),(x))
        d = dictmap(d_f)

        summary = lambda h, t: sum_for_publishers(packages, d, h, t)

    else:
        d_f = lambda x: ((x[4], x[1].id),(x))
        d = dictmap(d_f)

        summary = lambda h, t: sum_default(d, h, t)

    return summarise_results(conditions, mode, hierarchies,
                             tests, cdtns, indicators,
                             indicators_tests, packages, d, summary)

def agr_results(data, conditions=None, mode=None):
    def replace_first(tupl, newval):
        return tuple([newval] + list(tupl)[1:])
    switch_first = lambda t: replace_first(t, t[0].as_dict())
    fixed_data = map(switch_first, data)
    results = _agr_results(fixed_data, conditions, mode)
    return results
<commit_msg>Summary provides longdescriptions<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def publisher_mode(mode):
    if mode in ["publisher", "publisher_simple", "publisher_indicators"]:
        return True
    return False

def publisher_indicators(indicators, indicators_tests, simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    indicators_out = {}
    for indicator, indicatordata in indicators:
        indicators_out[indicator] = {}
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for test, testdata in simple_out.items():
            try:
                testing = (indicator, test)
                if (testing in indicators_tests):
                    results_pct+= simple_out[test]["results_pct"]
                    results_num+= simple_out[test]["results_num"]
                    results_weighted_pct_average_numerator += (simple_out[test]["results_pct"]*simple_out[test]["results_num"])
                    oktest = test
                    indicator_test_data.append(simple_out[test])
            except KeyError:
                pass
        indicators_out[indicator] = {
            "indicator": {
                "id": indicator,
                "name": indicatordata[0],
                "description": indicatordata[1],
                "indicator_type": indicatordata[2],
                "indicator_category_name": indicatordata[3],
                "indicator_subcategory_name": indicatordata[4],
                "longdescription": indicatordata[5]
                },
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }
    return indicators_out
    #for test, testdata in simple_out.items():
    #    indicators_out[testdata["indicator"]["id"]] = testdata
    #return indicators_out

def make_summary(test_id, test_description, test_group,
                 results_pct, results_num):
    return {
        "test": {
            "id": test_id,
            "description": test_description,
            "test_group": test_group
            },
        "results_pct": results_pct,
        "results_num": results_num
        }

def publisher_simple(out, cdtns):
    simple_out = {}
    hierarchies = set(out)
    tests = set()
    for h in hierarchies:
        tests.update(set(out[h]))
    for t in tests:
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0
        for hierarchy in hierarchies:
            try:
                key = (t,'activity hierarchy', str(hierarchy))
                try:
                    if ((cdtns) and (key in cdtns) and (cdtns[key][0]==0)):
                        continue
                except KeyError:
                    pass
                results_pct+= out[hierarchy][t]["results_pct"]
                results_num+= out[hierarchy][t]["results_num"]
                results_weighted_pct_average_numerator += (out[hierarchy][t]["results_pct"]*out[hierarchy][t]["results_num"])
                okhierarchy = hierarchy
            except KeyError:
                pass

        tmp = make_summary(
            out[okhierarchy][t]['test']["id"],
            out[okhierarchy][t]['test']["description"],
            out[okhierarchy][t]['test']["test_group"],
            (results_weighted_pct_average_numerator/results_num),
            results_num
            )
        tmp["indicator"] = out[okhierarchy][t]['indicator']
        simple_out[t] = tmp
    return simple_out

def sum_for_publishers(packages, d, h, t):
    # aggregate data across multiple packages for a single publisher

    # for each package, add percentage for each
    total_pct = 0
    total_activities = 0

    # need below to only include packages that are in this hierarchy
    packages_in_hierarchy = 0

    relevant = lambda p: (h, t, p) in d
    relevant_data = map(lambda p: d[(h, t, p)], filter(relevant, packages))

    pct = lambda i: i[2]
    activities = lambda i: i[3]

    try:
        total_pct = reduce(operator.add, map(pct, relevant_data))
        total_activities = reduce(operator.add, map(activities, relevant_data))
    except Exception:
        total_pct = 0.0
        total_activities = 0
    packages_in_hierarchy = len(relevant_data)

    if total_activities <= 0:
        return {}

    ok_tdata = relevant_data[-1] ## FIXME: this is obviously wrong

    tmp = make_summary(
        ok_tdata[1].id,
        ok_tdata[1].description,
        ok_tdata[1].test_group,
        int(float(total_pct/packages_in_hierarchy)),
        total_activities
        )
    tmp["indicator"] = ok_tdata[0]
    tmp["result_hierarchy"] = total_activities
    return tmp

def sum_default(d, h, t):
    # return data for this single package
    data = d.get((h, t), None)
    if data is None:
        return None

    tmp = make_summary(
        data[1].id,
        data[1].description,
        data[1].test_group,
        data[2],
        data[3])
    tmp["result_hierarchy"] = data[4]
    return tmp

def summarise_results(conditions, mode, hierarchies,
                      tests, cdtns, indicators,
                      indicators_tests, packages, d, summary):

    def summaries(summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def add_condition(i):
        h, t, tdata = i
        if conditions:
            key = (t, 'activity hierarchy', str(h))
            if key in cdtns:
                tdata["condition"] = cdtns[key]
        return h, t, tdata

    summaries = (add_condition(i) for i in summaries(summary))

    ## FIXME: the bit which removes empty data seems to have been deleted

    tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])
    out = reform_dict(tmp_out)

    if mode not in ["publisher_simple", "publisher_indicators"]:
        return out

    simple_out = publisher_simple(out, cdtns)

    if mode != "publisher_indicators":
        return simple_out

    return publisher_indicators(indicators, indicators_tests, simple_out)

def _agr_results(data, conditions=None, mode=None):
    """
        data variable looks like this:
            models.Indicator,
            models.Test,
            models.AggregateResult.results_data, ## percentage
            models.AggregateResult.results_num, ## total values
            models.AggregateResult.result_hierarchy

        conditions variable looks like this:
            models.PublisherCondition.query.filter_by(publisher_id=p[1].id).all()

            ======================================
            id = Column(Integer, primary_key=True)
            publisher_id = Column(Integer, ForeignKey('packagegroup.id'))
            test_id = Column(Integer, ForeignKey('test.id'))
            operation = Column(Integer) # show (1) or don't show (0) result
            condition = Column(UnicodeText) # activity level, hierarchy 2
            condition_value = Column(UnicodeText) # True, 2, etc.
            description = Column(UnicodeText)
            file = Column(UnicodeText)
            line = Column(Integer)
            active = Column(Boolean)
            ======================================
        mode can be:
            None = package-specific
            "publisher" = aggregate results for a whole publisher

        TODO:
            in publisher mode, allow weighting of data by package rather than number of activities.
    """

    def gen_hierarchies():
        for i in set(map(lambda x: (x[4]), data)):
            yield i
    hierarchies = gen_hierarchies()

    def gen_tests():
        for i in set(map(lambda x: (x[1].id), data)):
            yield i
    tests = gen_tests()

    cdtns = None
    if conditions:
        cdtns = dict(map(lambda x: (
                    (x.test_id, x.condition, x.condition_value),
                    (x.operation, x.description)
                    ), conditions))

    def setmap(lam):
        return set(map(lam, data))

    def dictmap(lam):
        return dict(map(lam, data))

    if publisher_mode(mode):
        ind_f = lambda x: (x[0]["id"], (x[0]["name"], x[0]["description"], x[0]["indicator_type"], x[0]["indicator_category_name"], x[0]["indicator_subcategory_name"], x[0]["longdescription"]))
        indicators = setmap(ind_f)

        ind_test_f = lambda x: (x[0]["id"], x[1].id)
        indicators_tests = list(setmap(ind_test_f))

        pkg_f = lambda x: x[5]
        packages = setmap(pkg_f)

        d_f = lambda x: ((x[4], x[1].id, x[5]),(x))
        d = dictmap(d_f)

        summary = lambda h, t: sum_for_publishers(packages, d, h, t)

    else:
        d_f = lambda x: ((x[4], x[1].id),(x))
        d = dictmap(d_f)

        summary = lambda h, t: sum_default(d, h, t)

    return summarise_results(conditions, mode, hierarchies,
                             tests, cdtns, indicators,
                             indicators_tests, packages, d, summary)

def agr_results(data, conditions=None, mode=None):
    def replace_first(tupl, newval):
        return tuple([newval] + list(tupl)[1:])
    switch_first = lambda t: replace_first(t, t[0].as_dict())
    fixed_data = map(switch_first, data)
    results = _agr_results(fixed_data, conditions, mode)
    return results
<|endoftext|>


<commit_before>#!/usr/bin/env python
import weakref
import dbus
import ibus
from ibus import keysyms
from clientmanager import ClientManager
from factorymanager import FactoryManager
from connection import Connection
from panel import Panel, DummyPanel

class IBus (ibus.Object):
	def __init__ (self):
		ibus.Object.__init__ (self)
		self._connections = {}
		self._client_manager = ClientManager ()
		self._factory_manager = FactoryManager ()
		self._panel = DummyPanel ()

		self._focused_client = None
		self._last_focused_client = None
		self._client_handlers = []

		self._last_key = None

	def new_connection (self, dbusconn):
		assert dbusconn not in self._connections
		self._connections[dbusconn] = Connection (dbusconn)

	def remove_connection (self, dbusconn):
		assert dbusconn in self._connections

		# destroy the connection
		self._connections[dbusconn].destroy ()
		del self._connections[dbusconn]

	def _lookup_ibus_connection (self, dbusconn):
		if dbusconn not in self._connections:
			raise ibus.IBusException ("can not find ibus.Connection")
		return self._connections[dbusconn]

	##########################################################
	# methods for im client
	##########################################################
	def register_client (self, name, dbusconn):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		client = self._client_manager.register_client (name, ibusconn)
		client.connect ("destroy", self._client_destroy_cb)
		factory = self._factory_manager.get_default_factory ()
		if factory:
			engine = factory.create_engine ()
			client.set_engine (engine)

	def focus_in (self, dbusconn):
		client = self._lookup_client (dbusconn)

		if self._focused_client != client and self._focused_client != None:
			map (self._focused_client.disconnect, self._client_handlers)
			del self._client_handlers[:]
			self._focused_client.focus_out ()

		# Install all callback functions
		id = client.connect ("update-preedit", self._update_preedit_cb)
		self._client_handlers.append (id)
		id = client.connect ("update-aux-string", self._update_aux_string_cb)
		self._client_handlers.append (id)
		id = client.connect ("update-lookup-table", self._update_lookup_table_cb)
		self._client_handlers.append (id)

		self._panel.reset ()
		self._focused_client = client
		self._last_focused_client = client
		client.focus_in ()

	def focus_out (self, dbusconn):
		client = self._lookup_client (dbusconn)
		if client == self._focused_client:
			map (self._focused_client.disconnect, self._client_handlers)
			del self._client_handlers[:]
			self._focused_client = None
		client.focus_out ()
		self._panel.reset ()

	def reset (self, dbusconn):
		client = self._lookup_client (dbusconn)
		self._panel.reset ()
		client.reset ()

	def is_enabled (self, dbusconn):
		client = self._lookup_client (dbusconn)
		return client.is_enabled ()

	def process_key_event (self, keyval, is_press, state,
								dbusconn, reply_cb, error_cb):
		client = self._lookup_client (dbusconn)

		if self._filter_hotkeys (client, keyval, is_press, state):
			reply_cb (True)
			return
		else:
			client.process_key_event (keyval, is_press, state, reply_cb, error_cb)

	def set_cursor_location (self, x, y, w, h, dbusconn):
		client = self._lookup_client (dbusconn)
		client.set_cursor_location (x, y, w, h)
		self._panel.set_cursor_location (x, y, w, h)

	def _filter_hotkeys (self, client, keyval, is_press, state):
		if is_press and keyval == keysyms.space \
			and state == keysyms.CONTROL_MASK:
			enable = not client.is_enabled ()
			client.set_enable (enable)
			if client.get_engine () == None and enable:
				factory = self._factory_manager.get_default_factory ()
				if factory:
					engine = factory.create_engine ()
					client.set_engine (engine)
			return True
		return False

	def _lookup_client (self, dbusconn):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		return self._client_manager.lookup_client (ibusconn)
		if dbusconn not in self._clients:
			raise ibus.IBusException ("not register the client")
		return self._clients[dbusconn]

	def _update_preedit_cb (self, client, text, attrs, cursor_pos, show):
		assert self._focused_client == client

		self._panel.update_preedit_string (text, attrs, cursor_pos, show)

	def _update_aux_string_cb (self, client, text, attrs, show):
		assert self._focused_client == client

		self._panel.update_aux_string (text, attrs, show)

	def _update_lookup_table_cb (self, client, lookup_table, show):
		assert self._focused_client == client

		self._panel.update_lookup_table (lookup_table, show)

	def _show_lookup_table_cb (self, client, lookup_table):
		assert self._focused_client == client

		self._panel.show_candidate_window ()

	def _hide_lookup_table_cb (self, client, lookup_table):
		assert self._focused_client == client

		self._panel.hide_candidate_window ()

	def _client_destroy_cb (self, client):
		if client == self._focused_client:
			del self._client_handlers[:]
			self._focused_client = None

	##########################################################
	# methods for im engines
	##########################################################
	def register_factories (self, object_paths, dbusconn):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		self._factory_manager.register_factories (object_paths, ibusconn)

	def dispatch_dbus_signal (self, dbusconn, message):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		ibusconn.dispatch_dbus_signal (message)

	def _lookup_engine (self, dbusconn, path):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		return self._factory_manager.lookup_engine (ibusconn, path)

	##########################################################
	# methods for panel
	##########################################################
	def register_panel (self, object_path, replace, dbusconn):
		if not isinstance (self._panel, DummyPanel) and replace == False:
			raise ibus.Exception ("has have a panel!")
		if not isinstance (self._panel, DummyPanel):
			self._panel.destroy ()
		ibusconn = self._lookup_ibus_connection (dbusconn)
		self._panel = Panel (ibusconn, object_path)
		self._panel.connect ("page-up", self._panel_page_up_cb)
		self._panel.connect ("page-down", self._panel_page_down_cb)
		self._panel.connect ("cursor-up", self._panel_cursor_up_cb)
		self._panel.connect ("cursor-down", self._panel_cursor_down_cb)
		self._panel.connect ("destroy", self._panel_destroy_cb)

	def _panel_page_up_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.page_up ()

	def _panel_page_down_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.page_down ()

	def _panel_cursor_up_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.cursor_up ()

	def _panel_cursor_down_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.cursor_down ()

	def _panel_destroy_cb (self, panel):
		assert panel == self._panel
		self._panel = DummyPanel ()

class IBusProxy (ibus.IIBus):
	SUPPORTS_MULTIPLE_CONNECTIONS = True

	def __init__ (self):
		ibus.IIBus.__init__ (self)
		self._ibus = IBus ()

	def new_connection (self, dbusconn):
		self._ibus.new_connection (dbusconn)

	def remove_connection (self, dbusconn):
		self._ibus.remove_connection (dbusconn)

	def dispatch_dbus_signal (self, dbusconn, message):
		return self._ibus.dispatch_dbus_signal (dbusconn, message)

	def GetIBusAddress (self, dbusconn):
		return self._ibus_addr

	def RegisterClient (self, client_name, dbusconn):
		self._ibus.register_client (client_name, dbusconn)

	def RegisterFactories (self, object_paths, dbusconn):
		self._ibus.register_factories (object_paths, dbusconn)

	def UnregisterEngines (self, object_paths, dbusconn):
		self._ibus.unregister_engines (object_paths, dbusconn)

	def RegisterPanel (self, object_path, replace, dbusconn):
		self._ibus.register_panel (object_path, replace, dbusconn)

	def ProcessKeyEvent (self, keyval, is_press, state, \
							dbusconn, reply_cb, error_cb):
		try:
			self._ibus.process_key_event (keyval, is_press, state,
							dbusconn, reply_cb, error_cb)
		except Exception, e:
			error_cb (e)

	def SetCursorLocation (self, x, y, w, h, dbusconn):
		self._ibus.set_cursor_location (x, y, w, h, dbusconn)

	def FocusIn (self, dbusconn):
		self._ibus.focus_in (dbusconn)

	def FocusOut (self, dbusconn):
		self._ibus.focus_out (dbusconn)

	def Reset (self, dbusconn):
		self._ibus.reset (dbusconn)

	def IsEnabled (self, dbusconn):
		return self._ibus.is_enabled (dbusconn)

<commit_msg>Fix problem when a new panel replacing the old one.<commit_after>#!/usr/bin/env python
import weakref
import dbus
import ibus
from ibus import keysyms
from clientmanager import ClientManager
from factorymanager import FactoryManager
from connection import Connection
from panel import Panel, DummyPanel

class IBus (ibus.Object):
	def __init__ (self):
		ibus.Object.__init__ (self)
		self._connections = {}
		self._client_manager = ClientManager ()
		self._factory_manager = FactoryManager ()
		self._panel = DummyPanel ()

		self._focused_client = None
		self._last_focused_client = None
		self._client_handlers = []

		self._last_key = None

	def new_connection (self, dbusconn):
		assert dbusconn not in self._connections
		self._connections[dbusconn] = Connection (dbusconn)

	def remove_connection (self, dbusconn):
		assert dbusconn in self._connections

		# destroy the connection
		self._connections[dbusconn].destroy ()
		del self._connections[dbusconn]

	def _lookup_ibus_connection (self, dbusconn):
		if dbusconn not in self._connections:
			raise ibus.IBusException ("can not find ibus.Connection")
		return self._connections[dbusconn]

	##########################################################
	# methods for im client
	##########################################################
	def register_client (self, name, dbusconn):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		client = self._client_manager.register_client (name, ibusconn)
		client.connect ("destroy", self._client_destroy_cb)
		factory = self._factory_manager.get_default_factory ()
		if factory:
			engine = factory.create_engine ()
			client.set_engine (engine)

	def focus_in (self, dbusconn):
		client = self._lookup_client (dbusconn)

		if self._focused_client != client and self._focused_client != None:
			map (self._focused_client.disconnect, self._client_handlers)
			del self._client_handlers[:]
			self._focused_client.focus_out ()

		# Install all callback functions
		id = client.connect ("update-preedit", self._update_preedit_cb)
		self._client_handlers.append (id)
		id = client.connect ("update-aux-string", self._update_aux_string_cb)
		self._client_handlers.append (id)
		id = client.connect ("update-lookup-table", self._update_lookup_table_cb)
		self._client_handlers.append (id)

		self._panel.reset ()
		self._focused_client = client
		self._last_focused_client = client
		client.focus_in ()

	def focus_out (self, dbusconn):
		client = self._lookup_client (dbusconn)
		if client == self._focused_client:
			map (self._focused_client.disconnect, self._client_handlers)
			del self._client_handlers[:]
			self._focused_client = None
		client.focus_out ()
		self._panel.reset ()

	def reset (self, dbusconn):
		client = self._lookup_client (dbusconn)
		self._panel.reset ()
		client.reset ()

	def is_enabled (self, dbusconn):
		client = self._lookup_client (dbusconn)
		return client.is_enabled ()

	def process_key_event (self, keyval, is_press, state,
								dbusconn, reply_cb, error_cb):
		client = self._lookup_client (dbusconn)

		if self._filter_hotkeys (client, keyval, is_press, state):
			reply_cb (True)
			return
		else:
			client.process_key_event (keyval, is_press, state, reply_cb, error_cb)

	def set_cursor_location (self, x, y, w, h, dbusconn):
		client = self._lookup_client (dbusconn)
		client.set_cursor_location (x, y, w, h)
		self._panel.set_cursor_location (x, y, w, h)

	def _filter_hotkeys (self, client, keyval, is_press, state):
		if is_press and keyval == keysyms.space \
			and state == keysyms.CONTROL_MASK:
			enable = not client.is_enabled ()
			client.set_enable (enable)
			if client.get_engine () == None and enable:
				factory = self._factory_manager.get_default_factory ()
				if factory:
					engine = factory.create_engine ()
					client.set_engine (engine)
			return True
		return False

	def _lookup_client (self, dbusconn):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		return self._client_manager.lookup_client (ibusconn)
		if dbusconn not in self._clients:
			raise ibus.IBusException ("not register the client")
		return self._clients[dbusconn]

	def _update_preedit_cb (self, client, text, attrs, cursor_pos, show):
		assert self._focused_client == client

		self._panel.update_preedit_string (text, attrs, cursor_pos, show)

	def _update_aux_string_cb (self, client, text, attrs, show):
		assert self._focused_client == client

		self._panel.update_aux_string (text, attrs, show)

	def _update_lookup_table_cb (self, client, lookup_table, show):
		assert self._focused_client == client

		self._panel.update_lookup_table (lookup_table, show)

	def _show_lookup_table_cb (self, client, lookup_table):
		assert self._focused_client == client

		self._panel.show_candidate_window ()

	def _hide_lookup_table_cb (self, client, lookup_table):
		assert self._focused_client == client

		self._panel.hide_candidate_window ()

	def _client_destroy_cb (self, client):
		if client == self._focused_client:
			del self._client_handlers[:]
			self._focused_client = None

	##########################################################
	# methods for im engines
	##########################################################
	def register_factories (self, object_paths, dbusconn):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		self._factory_manager.register_factories (object_paths, ibusconn)

	def dispatch_dbus_signal (self, dbusconn, message):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		ibusconn.dispatch_dbus_signal (message)

	def _lookup_engine (self, dbusconn, path):
		ibusconn = self._lookup_ibus_connection (dbusconn)
		return self._factory_manager.lookup_engine (ibusconn, path)

	##########################################################
	# methods for panel
	##########################################################
	def register_panel (self, object_path, replace, dbusconn):
		if not isinstance (self._panel, DummyPanel) and replace == False:
			raise ibus.Exception ("has have a panel!")
		if not isinstance (self._panel, DummyPanel):
			self._panel.destroy ()
		ibusconn = self._lookup_ibus_connection (dbusconn)
		self._panel = Panel (ibusconn, object_path)
		self._panel.connect ("page-up", self._panel_page_up_cb)
		self._panel.connect ("page-down", self._panel_page_down_cb)
		self._panel.connect ("cursor-up", self._panel_cursor_up_cb)
		self._panel.connect ("cursor-down", self._panel_cursor_down_cb)
		self._panel.connect ("destroy", self._panel_destroy_cb)

	def _panel_page_up_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.page_up ()

	def _panel_page_down_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.page_down ()

	def _panel_cursor_up_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.cursor_up ()

	def _panel_cursor_down_cb (self, panel):
		assert panel == self._panel
		if self._focused_client:
			self._focused_client.cursor_down ()

	def _panel_destroy_cb (self, panel):
		if panel == self._panel:
			self._panel = DummyPanel ()

class IBusProxy (ibus.IIBus):
	SUPPORTS_MULTIPLE_CONNECTIONS = True

	def __init__ (self):
		ibus.IIBus.__init__ (self)
		self._ibus = IBus ()

	def new_connection (self, dbusconn):
		self._ibus.new_connection (dbusconn)

	def remove_connection (self, dbusconn):
		self._ibus.remove_connection (dbusconn)

	def dispatch_dbus_signal (self, dbusconn, message):
		return self._ibus.dispatch_dbus_signal (dbusconn, message)

	def GetIBusAddress (self, dbusconn):
		return self._ibus_addr

	def RegisterClient (self, client_name, dbusconn):
		self._ibus.register_client (client_name, dbusconn)

	def RegisterFactories (self, object_paths, dbusconn):
		self._ibus.register_factories (object_paths, dbusconn)

	def UnregisterEngines (self, object_paths, dbusconn):
		self._ibus.unregister_engines (object_paths, dbusconn)

	def RegisterPanel (self, object_path, replace, dbusconn):
		self._ibus.register_panel (object_path, replace, dbusconn)

	def ProcessKeyEvent (self, keyval, is_press, state, \
							dbusconn, reply_cb, error_cb):
		try:
			self._ibus.process_key_event (keyval, is_press, state,
							dbusconn, reply_cb, error_cb)
		except Exception, e:
			error_cb (e)

	def SetCursorLocation (self, x, y, w, h, dbusconn):
		self._ibus.set_cursor_location (x, y, w, h, dbusconn)

	def FocusIn (self, dbusconn):
		self._ibus.focus_in (dbusconn)

	def FocusOut (self, dbusconn):
		self._ibus.focus_out (dbusconn)

	def Reset (self, dbusconn):
		self._ibus.reset (dbusconn)

	def IsEnabled (self, dbusconn):
		return self._ibus.is_enabled (dbusconn)

<|endoftext|>


<commit_before>#!/usr/bin/python

import argparse
import os
import re
import subprocess

REDIS_HOST = ''

RPC_BACKEND = re.compile('^\s*rpc_backend')
DEFAULT = re.compile('^\s*\[DEFAULT\]\s*$')
REDIS_SECTION = re.compile('^\s*\[matchmaker_redis\]\s*$')
ZMQ_SECTION = re.compile('^\s*\[oslo_messaging_zmq\]\s*$')

IGNORE = ['debug', 'rpc_backend', 'rpc_zmq_matchmaker', 'rpc_zmq_host',
          'default_log_levels', 'sentinel_hosts', 'use_router_proxy',
          'rpc_use_acks', 'host']

FRONTEND_PORT = 50001
BACKEND_PORT = 50002
PUBLISHER_PORT = 50003

def get_command_output(cmd):
    print 'Executing cmd: %s' % cmd
    pp = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    outp, err = pp.communicate()

    if pp.returncode != 0:
        print ('RuntimeError: Process returned non-zero code %i' % pp.returncode)

    return outp.strip()

def generate_proxy_conf(use_pub_sub):
    print get_command_output("rm -rf /etc/zmq-proxy/")
    print get_command_output("mkdir /etc/zmq-proxy/")
    with open('/etc/zmq-proxy/zmq.conf', 'w+') as conf_f:
        conf_f.write("[oslo_messaging_zmq]\n"
                     "rpc_zmq_host=%s\n"
                     "use_pub_sub=%s\n"
                     "[matchmaker_redis]\n"
                     "host=%s" % (get_command_output("hostname"),
                                  "true" if use_pub_sub else "false",
                                  get_managable_ip_from_node(REDIS_HOST)))

def start_proxy(debug, use_pub_sub, double_router):
    generate_proxy_conf(use_pub_sub)
    print get_command_output("rm -rf /var/log/zmq-proxy.log")
    print get_command_output("nohup /tmp/venv/bin/python /tmp/oslo.messaging/oslo_messaging/_cmd/zmq_proxy.py %(debug)s "
                             "--frontend-port %(fe)s %(backend_port)s --publisher-port %(pub)s "
                             "--config-file=/etc/zmq-proxy/zmq.conf "
                             "> /var/log/zmq-proxy.log 2>&1 < /var/log/zmq-proxy.log &" %
                             {"debug": "--debug" if debug else "",
                              "fe": FRONTEND_PORT,
                              "pub": PUBLISHER_PORT,
                              "backend_port": "--backend-port %s" % BACKEND_PORT if double_router else ""})

def kill_proxy():
    p_ids = get_command_output("ps aux | grep zm[q] | awk '{ print $2 }'").split('\n')
    for pid in p_ids:
        get_command_output("kill -9 %s" % pid)

def get_managable_ip_from_node(node):
    return get_command_output("host %s" % node).split(' ')[-1]

def hack_redis():
    file_name = '/etc/redis/redis.conf'
    print "Rewriting redis config %s" % file_name
    with open(file_name, 'r') as fl:
        content = fl.readlines()

    if not os.path.isfile(file_name+".backup"):
        with open(file_name+".backup", 'w') as fl:
            fl.write(''.join(content))

    newcontent = []
    for line in content:
        if line.startswith('bind 127.0.0.1'):
            replacement = 'bind 127.0.0.1 ' + get_managable_ip_from_node(REDIS_HOST)
            print line + '->' + replacement + '\n'
            newcontent.append(replacement)
        else:
            newcontent.append(line)

    with open(file_name, 'w') as fl:
        fl.write(''.join(newcontent))

def restore_backup():
    file_name = args.file_name
    backup_file_name = file_name+".backup"
    if not os.path.isfile(backup_file_name):
        raise RuntimeWarning("No backup found for %s. Operation is skipped." % file_name)

    if os.path.isfile(file_name):
        print get_command_output("rm -rf %s" % file_name)

    print get_command_output("mv %s %s" % (backup_file_name, file_name))

def hack_services(debug, use_acks, use_router_proxy):

    file_name = args.file_name
    with open(file_name, 'r') as fl:
        content = fl.readlines()

    if not os.path.isfile(file_name+".backup"):
        with open(file_name+".backup", 'w') as fl:
            fl.write(''.join(content))

    newcontent = []
    time_to_put_config = False
    for line in content:
        ignore = False
        for prefix in IGNORE:
            if line.startswith(prefix):
                ignore = True

        if ignore:
            continue

        if time_to_put_config:
            time_to_put_config = False

            if debug:
                newcontent.append('debug = True\n')

            newcontent.append('default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,iso8601=WARN,keystonemiddleware=WARN,oslo.messaging=%(debug)s,oslo_messaging=%(debug)s,qpid=WARN,requests.packages.urllib3.connectionpool=WARN,requests.packages.urllib3.util.retry=WARN,routes.middleware=WARN,sqlalchemy=WARN,stevedore=WARN,suds=INFO,taskflow=WARN,urllib3.connectionpool=WARN,urllib3.util.retry=WARN,websocket=WARN\n' %
                              {"debug": "DEBUG" if debug else "WARN"})

            newcontent.append('rpc_backend = zmq\n')

        if RPC_BACKEND.match(line) or REDIS_SECTION.match(line) or ZMQ_SECTION.match(line):
            continue

        if DEFAULT.match(line):
            time_to_put_config = True

        newcontent.append(line)

    newcontent.append('[oslo_messaging_zmq]\n')
    newcontent.append('rpc_zmq_host = %s\n' % get_command_output("hostname"))

    if not use_router_proxy:
        newcontent.append('zmq_linger = 20\n')

    newcontent.append('use_router_proxy = %s\n' % ("true" if use_router_proxy else "false"))
    newcontent.append('rpc_use_acks = %s\n' % ("true" if use_acks else "false"))
    newcontent.append('rpc_zmq_matchmaker = redis\n')
    newcontent.append('[matchmaker_redis]\n')
    newcontent.append('host=%s\n' % REDIS_HOST)

    with open(file_name, 'w') as fl:
        fl.write(''.join(newcontent))

parser = argparse.ArgumentParser()
parser.add_argument('--generate', dest='generate', action='store_true')
parser.add_argument('--start-proxy', dest='start_proxy', action='store_true')
parser.add_argument('--double-proxy', dest='double_proxy', action='store_true')
parser.add_argument('--kill-proxy', dest='kill_proxy', action='store_true')
parser.add_argument('--debug', dest='debug', action='store_true')
parser.add_argument('--use-acks', dest='use_acks', action='store_true')
parser.add_argument('--hack', dest='hack', action='store_true')
parser.add_argument('--restore-backup', dest='restore_backup', action='store_true')
parser.add_argument('--hack-redis', dest='hack_redis', action='store_true')
parser.add_argument('--use-pub-sub', dest='use_pub_sub', action='store_true')
parser.add_argument('--use-router-proxy', dest='use_router_proxy', action='store_true')
parser.add_argument('--file', dest='file_name', type=str)
parser.add_argument('--redis-host', dest='redis_host', type=str)
args = parser.parse_args()

if __name__ == "__main__":
    global REDIS_HOST
    try:
        REDIS_HOST = args.redis_host

        if args.generate:
            use_pub_sub = True if args.use_pub_sub else False
            generate_proxy_conf(use_pub_sub)
        elif args.hack:
            hack_services(args.debug, args.use_acks, args.use_router_proxy)
        elif args.restore_backup:
            restore_backup()
        elif args.hack_redis:
            hack_redis()
        elif args.kill_proxy:
            kill_proxy()
        elif args.start_proxy:
            start_proxy(args.debug, args.use_pub_sub, args.double_proxy)

    except RuntimeError as e:
        print(str(e))
<commit_msg>Use pub sub<commit_after>#!/usr/bin/python

import argparse
import os
import re
import subprocess

REDIS_HOST = ''

RPC_BACKEND = re.compile('^\s*rpc_backend')
DEFAULT = re.compile('^\s*\[DEFAULT\]\s*$')
REDIS_SECTION = re.compile('^\s*\[matchmaker_redis\]\s*$')
ZMQ_SECTION = re.compile('^\s*\[oslo_messaging_zmq\]\s*$')

IGNORE = ['debug', 'rpc_backend', 'rpc_zmq_matchmaker', 'rpc_zmq_host',
          'default_log_levels', 'sentinel_hosts', 'use_router_proxy',
          'rpc_use_acks', 'host']

FRONTEND_PORT = 50001
BACKEND_PORT = 50002
PUBLISHER_PORT = 50003

def get_command_output(cmd):
    print 'Executing cmd: %s' % cmd
    pp = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    outp, err = pp.communicate()

    if pp.returncode != 0:
        print ('RuntimeError: Process returned non-zero code %i' % pp.returncode)

    return outp.strip()

def generate_proxy_conf(use_pub_sub):
    print get_command_output("rm -rf /etc/zmq-proxy/")
    print get_command_output("mkdir /etc/zmq-proxy/")
    with open('/etc/zmq-proxy/zmq.conf', 'w+') as conf_f:
        conf_f.write("[oslo_messaging_zmq]\n"
                     "rpc_zmq_host=%s\n"
                     "use_pub_sub=%s\n"
                     "[matchmaker_redis]\n"
                     "host=%s" % (get_command_output("hostname"),
                                  "true" if use_pub_sub else "false",
                                  get_managable_ip_from_node(REDIS_HOST)))

def start_proxy(debug, use_pub_sub, double_router):
    generate_proxy_conf(use_pub_sub)
    print get_command_output("rm -rf /var/log/zmq-proxy.log")
    print get_command_output("nohup /tmp/venv/bin/python /tmp/oslo.messaging/oslo_messaging/_cmd/zmq_proxy.py %(debug)s "
                             "--frontend-port %(fe)s %(backend_port)s --publisher-port %(pub)s "
                             "--config-file=/etc/zmq-proxy/zmq.conf "
                             "> /var/log/zmq-proxy.log 2>&1 < /var/log/zmq-proxy.log &" %
                             {"debug": "--debug" if debug else "",
                              "fe": FRONTEND_PORT,
                              "pub": PUBLISHER_PORT,
                              "backend_port": "--backend-port %s" % BACKEND_PORT if double_router else ""})

def kill_proxy():
    p_ids = get_command_output("ps aux | grep zm[q] | awk '{ print $2 }'").split('\n')
    for pid in p_ids:
        get_command_output("kill -9 %s" % pid)

def get_managable_ip_from_node(node):
    return get_command_output("host %s" % node).split(' ')[-1]

def hack_redis():
    file_name = '/etc/redis/redis.conf'
    print "Rewriting redis config %s" % file_name
    with open(file_name, 'r') as fl:
        content = fl.readlines()

    if not os.path.isfile(file_name+".backup"):
        with open(file_name+".backup", 'w') as fl:
            fl.write(''.join(content))

    newcontent = []
    for line in content:
        if line.startswith('bind 127.0.0.1'):
            replacement = 'bind 127.0.0.1 ' + get_managable_ip_from_node(REDIS_HOST)
            print line + '->' + replacement + '\n'
            newcontent.append(replacement)
        else:
            newcontent.append(line)

    with open(file_name, 'w') as fl:
        fl.write(''.join(newcontent))

def restore_backup():
    file_name = args.file_name
    backup_file_name = file_name+".backup"
    if not os.path.isfile(backup_file_name):
        raise RuntimeWarning("No backup found for %s. Operation is skipped." % file_name)

    if os.path.isfile(file_name):
        print get_command_output("rm -rf %s" % file_name)

    print get_command_output("mv %s %s" % (backup_file_name, file_name))

def hack_services(debug, use_acks, use_router_proxy, use_pub_sub):

    file_name = args.file_name
    with open(file_name, 'r') as fl:
        content = fl.readlines()

    if not os.path.isfile(file_name+".backup"):
        with open(file_name+".backup", 'w') as fl:
            fl.write(''.join(content))

    newcontent = []
    time_to_put_config = False
    for line in content:
        ignore = False
        for prefix in IGNORE:
            if line.startswith(prefix):
                ignore = True

        if ignore:
            continue

        if time_to_put_config:
            time_to_put_config = False

            if debug:
                newcontent.append('debug = True\n')

            newcontent.append('default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,iso8601=WARN,keystonemiddleware=WARN,oslo.messaging=%(debug)s,oslo_messaging=%(debug)s,qpid=WARN,requests.packages.urllib3.connectionpool=WARN,requests.packages.urllib3.util.retry=WARN,routes.middleware=WARN,sqlalchemy=WARN,stevedore=WARN,suds=INFO,taskflow=WARN,urllib3.connectionpool=WARN,urllib3.util.retry=WARN,websocket=WARN\n' %
                              {"debug": "DEBUG" if debug else "WARN"})

            newcontent.append('rpc_backend = zmq\n')

        if RPC_BACKEND.match(line) or REDIS_SECTION.match(line) or ZMQ_SECTION.match(line):
            continue

        if DEFAULT.match(line):
            time_to_put_config = True

        newcontent.append(line)

    newcontent.append('[oslo_messaging_zmq]\n')
    newcontent.append('rpc_zmq_host = %s\n' % get_command_output("hostname"))

    if not use_router_proxy:
        newcontent.append('zmq_linger = 20\n')

    newcontent.append('use_router_proxy = %s\n' % ("true" if use_router_proxy else "false"))
    newcontent.append('use_pub_sub = %s\n' % ("true" if use_pub_sub else "false"))
    newcontent.append('rpc_use_acks = %s\n' % ("true" if use_acks else "false"))
    newcontent.append('rpc_zmq_matchmaker = redis\n')
    newcontent.append('[matchmaker_redis]\n')
    newcontent.append('host=%s\n' % REDIS_HOST)

    with open(file_name, 'w') as fl:
        fl.write(''.join(newcontent))

parser = argparse.ArgumentParser()
parser.add_argument('--generate', dest='generate', action='store_true')
parser.add_argument('--start-proxy', dest='start_proxy', action='store_true')
parser.add_argument('--double-proxy', dest='double_proxy', action='store_true')
parser.add_argument('--kill-proxy', dest='kill_proxy', action='store_true')
parser.add_argument('--debug', dest='debug', action='store_true')
parser.add_argument('--use-acks', dest='use_acks', action='store_true')
parser.add_argument('--hack', dest='hack', action='store_true')
parser.add_argument('--restore-backup', dest='restore_backup', action='store_true')
parser.add_argument('--hack-redis', dest='hack_redis', action='store_true')
parser.add_argument('--use-pub-sub', dest='use_pub_sub', action='store_true')
parser.add_argument('--use-router-proxy', dest='use_router_proxy', action='store_true')
parser.add_argument('--file', dest='file_name', type=str)
parser.add_argument('--redis-host', dest='redis_host', type=str)
args = parser.parse_args()

if __name__ == "__main__":
    global REDIS_HOST
    try:
        REDIS_HOST = args.redis_host

        if args.generate:
            use_pub_sub = True if args.use_pub_sub else False
            generate_proxy_conf(use_pub_sub)
        elif args.hack:
            hack_services(args.debug, args.use_acks, args.use_router_proxy, args.use_pub_sub)
        elif args.restore_backup:
            restore_backup()
        elif args.hack_redis:
            hack_redis()
        elif args.kill_proxy:
            kill_proxy()
        elif args.start_proxy:
            start_proxy(args.debug, args.use_pub_sub, args.double_proxy)

    except RuntimeError as e:
        print(str(e))
<|endoftext|>