<commit_before>"""
Keyword Recommenders.
"""

import math
import random
from utils import timeit
from collections import defaultdict

class RandomRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.all_products = []

    def __str__(self):
        return 'Random Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        # retrieve all products at once as there aren't many (< 4000)
        query = '''SELECT DISTINCT pageinfo FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND pageinfo != 'null' AND userid IN (
                SELECT user_id FROM %s
            )''' % query_train_table
        self.all_products = [(row['pageinfo'], 1.0) for row in self.dbm.get_rows(query)]

    def round_statistics(self):
        pass

    def recommend(self, query):
        return random.sample(self.all_products, self.limit)

class HottestRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.recommend_list = []

    def __str__(self):
        return 'Hottest Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        for row in self.dbm.get_rows('''SELECT pageinfo, COUNT(id) count FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND userid IN (
                SELECT user_id FROM %s
            ) GROUP BY pageinfo ORDER BY count DESC LIMIT %d''' % (query_train_table, self.limit)):
            self.recommend_list.append((row['pageinfo'], row['count']))
        #print self.recommend_list

    def round_statistics(self):
        pass

    def recommend(self, query):
        return self.recommend_list

class KeywordRecommender(object):
    def __init__(self, limit, dbm, ws, rm):
        """
        Make sure to source rec_tables.sql before using this class.
        @param dbm a DatabaseManager
        @param ws a WordSegmenter
        @param rm a RelevanceMeasure
        """
        self.limit = limit
        self.dbm = dbm
        self.ws = ws
        self.rm = rm
        self._related_product_cache = {}
        self._not_enough_recs = 0

    def __str__(self):
        return 'Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

    def use_keywords(self):
        return True

    def preprocess(self, query_train_table):
        self.query_train_table = query_train_table
        # empty cache so that cache from last round does not interfere with next round
        self._related_product_cache = {}
        self._not_enough_recs = 0

        self.dbm.begin()
        self.dbm.query('TRUNCATE TABLE keyword');
        self.dbm.query('TRUNCATE TABLE keyword_query');
        self.dbm.query('TRUNCATE TABLE keyword_product_weight');

        # these methods can be overridden by sub-classes
        self._build_keyword_product_mapping()
        self._build_product_keyword_mapping()
        self._measure_relevance()

        self.dbm.commit()

    @timeit
    def _build_keyword_product_mapping(self):
        self.keyword_count = defaultdict(int)
        self.keyword_product_count = defaultdict(lambda: defaultdict(int))
        for qrow in self.dbm.get_rows('SELECT id, query FROM %s' % self.query_train_table):
            # GROUP_CONCAT returns a comma-separeted string
            products = [(qprow['product_name'], qprow['sequences']) for qprow in self.dbm.get_rows('SELECT product_name, GROUP_CONCAT(sequence) AS sequences FROM query_product WHERE query_id = %s GROUP BY product_name', (qrow['id'],))]

            # remove duplicate
            keywords = set(self.ws.segment(qrow['query']))
            for kw in keywords:
                self.keyword_count[kw] += 1
                # store keyword-query relations in db
                self.dbm.insert('INSERT INTO keyword_query (keyword, query_id) VALUES (%s, %s)', (kw, qrow['id']))

            for p, sequences in products:
                # get product sequence in this session
                count = self.get_browse_count(sequences)
                # update keyword_product_count
                for kw in keywords:
                    self.keyword_product_count[kw][p] += count

    def get_browse_count(self, sequences):
        """Overrideable by sub-class.
        Multiple browses in a session always count 1."""
        return 1

    @timeit
    def _build_product_keyword_mapping(self):
        # construct product_keyword_count
        # it's actually equivalent to keyword_product_count, but can let compute
        # related_product_count faster
        self.product_keyword_count = defaultdict(dict)
        for kw, dt in self.keyword_product_count.iteritems():
            for p, c in dt.iteritems():
                self.product_keyword_count[p][kw] = c

    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            # will be used for statistics
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())

                # delegate to sub-classes
                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def round_statistics(self):
        """Get number of query, keywords, products, keyword-product relations of current round."""
        n_query = self.dbm.get_value("SELECT COUNT(*) FROM %s" % self.query_train_table)
        n_keyword = self.dbm.get_value("SELECT COUNT(*) FROM keyword")
        n_product = self.dbm.get_value("SELECT COUNT(DISTINCT product) FROM keyword_product_weight")
        n_relation = self.dbm.get_value("SELECT COUNT(*) FROM keyword_product_weight")

        print 'Round statistics: query: %d (not enough %d), keyword: %d, product: %d, relation: %d, A/M: %.2f%%' % (n_query, self._not_enough_recs, n_keyword, n_product, n_relation, 100.0*n_relation / (n_keyword*n_product))

    def recommend(self, query):
        keywords = self.ws.segment(query)
        product_weight = defaultdict(float)
        # gather product weights
        for kw in keywords:
            for product, weight in self.__fetch_related_products(kw):
                product_weight[product] += weight

        # convert dict to list for sorting
        product_weight_list = [item for item in product_weight.iteritems()]
        product_weight_list.sort(key=lambda t: t[1], reverse=True)

        if len(product_weight_list) < self.limit:
            self._not_enough_recs += 1

        return product_weight_list[:self.limit]

    def __fetch_related_products(self, keyword):
        if not self._related_product_cache.has_key(keyword):
            self._related_product_cache[keyword] = [(row['product'], row['weight']) for row in self.dbm.get_rows('SELECT product, weight FROM keyword_product_weight WHERE keyword = %s', (keyword,))]
        return self._related_product_cache[keyword]

class KeywordRecommenderHottestFallback(KeywordRecommender):
    """A recommender which uses KeywordRecommender's recommendations first,
    but turns to HottestRecommender if its recommendations are not enough."""

    def __init__(self, *args):
        """Identical to that of KeywordRecommender"""
        super(KeywordRecommenderHottestFallback, self).__init__(*args)
        self.hottest_recommender = HottestRecommender(*args)

    def __str__(self):
        return 'Keyword Recommender with Hottest Recommender fallback with %s[N=%d]' % (self.rm, self.limit)

    def preprocess(self, query_train_table):
        super(KeywordRecommenderHottestFallback, self).preprocess(query_train_table)
        self.hottest_recommender.preprocess(query_train_table)

    def recommend(self, query):
        recommendations = super(KeywordRecommenderHottestFallback, self).recommend(query)
        num_rec = len(recommendations)
        if num_rec == self.limit:
            return recommendations

        # ask HottestRecommender for more
        # note that create list in order not to break HottestRecommender.recommend_list
        hot_recommendations = self.hottest_recommender.recommend(query)[:self.limit-num_rec]

        # ensure hot_recommendations's weight is no greater than any from keyword recommendations
        max_hot_rec_weight = hot_recommendations[0][1]
        min_key_rec_weight = recommendations[-1][1] if num_rec > 0 else max_hot_rec_weight
        recommendations.extend((t[0], 1.0*min_key_rec_weight*t[1]/max_hot_rec_weight) for t in hot_recommendations)

        return recommendations

class LinearSequenceKeywordRecommender(KeywordRecommender):
    def _heuristic_weight(self, sequence):
        return -math.log(sequence+1, 2)/8.0 + 1.125

    def get_browse_count(self, sequences):
        return sum(self._heuristic_weight(int(seq)) for seq in sequences.split(','))

    def __str__(self):
        return 'Linear Sequenced Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

class WeightedSequenceRelevanceMixin(object):
    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())
                # get average sequence from database
                # TODO: very inefficient, get a group all average sequences for a keyword at once
                #avg_sequence = self.dbm.get_value('select avg(sequence) from query_product where product_name = %s AND query_id in (select query_id from keyword_query where keyword = %s)', (product, keyword))
                avg_sequence = 1

                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number, avg_sequence)

                # sub-class can override sequence_weight
                # relevance *= self.sequence_weight(avg_sequence)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def sequence_weight(self, avg_sequence):
        return 1

# ensure WSRM._measure_relevance will be called with putting it before KeywordRecommender
# ref: http://python-history.blogspot.com/2010/06/method-resolution-order.html
class SequenceKeywordRecommender(WeightedSequenceRelevanceMixin, LinearSequenceKeywordRecommender):
    """This recommender weights browse count by distribution of sequence."""

    @timeit
    def preprocess(self, query_train_table):
        # first, get sequence distribution
        max_occurrence = self.dbm.get_value('SELECT MAX(c) FROM (SELECT sequence, COUNT(sequence) c FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence) T' % query_train_table)
        self.sequence_dist = {row['sequence']: float(row['ratio']) for row in self.dbm.get_rows('SELECT sequence, COUNT(sequence)/%d ratio FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence' % (max_occurrence,query_train_table))}
        self.pivot_seq = max(self.sequence_dist.iteritems(), key=lambda t:t[1])[0]

        # then, call KeywordRecommender's preprocess
        super(SequenceKeywordRecommender, self).preprocess(query_train_table)

    def _heuristic_weight(self, sequence):
        weight = self.sequence_dist[sequence]
        if self.pivot_seq-sequence < 0:
            return weight
        return 1 + weight

    def __str__(self):
        return 'Sequenced Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

class RelevanceMeasure(object):
    """Defines the RelevanceMeasure interface."""

    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        """
        @param count number of times the keyword visit the product
        @param related_product_info the tuple (related_product_number, related_product_count)
        @param related_keyword_info the tuple (related_keyword_number, related_keyword_count)
        @param all_product_number number of all products
        """
        raise NotImplemented

class BCMeasure(RelevanceMeasure):
    def get_relevance(self, count, *ignored):
        return count

    def __str__(self):
        return 'BC'

class BCIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return count * ipf

    def __str__(self):
        return 'BC-IPF'

class BFIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        bf = 1.0 * count / related_keyword_info[1]
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return bf * ipf

    def __str__(self):
        return 'BF-IPF'

if __name__ == '__main__':
    import config
    from database import DatabaseManager
    from word_segment import SpaceWordSegmenter
    dbm = DatabaseManager(config.DB_HOST, config.DB_USER, config.DB_PASSWORD, config.DB_NAME)
    try:
        word_segmenter = SpaceWordSegmenter()
        rmeasure = BCIPFMeasure()
        recommender = KeywordRecommender(10, dbm, word_segmenter, rmeasure)
        recommender.preprocess('query_train')
    finally:
        dbm.close()
<commit_msg>try varisou sequence weighting measures<commit_after>"""
Keyword Recommenders.
"""

import math
import random
from utils import timeit
from collections import defaultdict

class RandomRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.all_products = []

    def __str__(self):
        return 'Random Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        # retrieve all products at once as there aren't many (< 4000)
        query = '''SELECT DISTINCT pageinfo FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND pageinfo != 'null' AND userid IN (
                SELECT user_id FROM %s
            )''' % query_train_table
        self.all_products = [(row['pageinfo'], 1.0) for row in self.dbm.get_rows(query)]

    def round_statistics(self):
        pass

    def recommend(self, query):
        return random.sample(self.all_products, self.limit)

class HottestRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.recommend_list = []

    def __str__(self):
        return 'Hottest Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        for row in self.dbm.get_rows('''SELECT pageinfo, COUNT(id) count FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND userid IN (
                SELECT user_id FROM %s
            ) GROUP BY pageinfo ORDER BY count DESC LIMIT %d''' % (query_train_table, self.limit)):
            self.recommend_list.append((row['pageinfo'], row['count']))
        #print self.recommend_list

    def round_statistics(self):
        pass

    def recommend(self, query):
        return self.recommend_list

class KeywordRecommender(object):
    def __init__(self, limit, dbm, ws, rm):
        """
        Make sure to source rec_tables.sql before using this class.
        @param dbm a DatabaseManager
        @param ws a WordSegmenter
        @param rm a RelevanceMeasure
        """
        self.limit = limit
        self.dbm = dbm
        self.ws = ws
        self.rm = rm
        self._related_product_cache = {}
        self._not_enough_recs = 0

    def __str__(self):
        return 'Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

    def use_keywords(self):
        return True

    def preprocess(self, query_train_table):
        self.query_train_table = query_train_table
        # empty cache so that cache from last round does not interfere with next round
        self._related_product_cache = {}
        self._not_enough_recs = 0

        self.dbm.begin()
        self.dbm.query('TRUNCATE TABLE keyword');
        self.dbm.query('TRUNCATE TABLE keyword_query');
        self.dbm.query('TRUNCATE TABLE keyword_product_weight');

        # these methods can be overridden by sub-classes
        self._build_keyword_product_mapping()
        self._build_product_keyword_mapping()
        self._measure_relevance()

        self.dbm.commit()

    @timeit
    def _build_keyword_product_mapping(self):
        self.keyword_count = defaultdict(int)
        self.keyword_product_count = defaultdict(lambda: defaultdict(int))
        for qrow in self.dbm.get_rows('SELECT id, query FROM %s' % self.query_train_table):
            # GROUP_CONCAT returns a comma-separeted string
            products = [(qprow['product_name'], qprow['sequences']) for qprow in self.dbm.get_rows('SELECT product_name, GROUP_CONCAT(sequence) AS sequences FROM query_product WHERE query_id = %s GROUP BY product_name', (qrow['id'],))]

            # remove duplicate
            keywords = set(self.ws.segment(qrow['query']))
            for kw in keywords:
                self.keyword_count[kw] += 1
                # store keyword-query relations in db
                self.dbm.insert('INSERT INTO keyword_query (keyword, query_id) VALUES (%s, %s)', (kw, qrow['id']))

            for p, sequences in products:
                # get product sequence in this session
                count = self.get_browse_count(sequences)
                # update keyword_product_count
                for kw in keywords:
                    self.keyword_product_count[kw][p] += count

    def get_browse_count(self, sequences):
        """Overrideable by sub-class.
        Multiple browses in a session always count 1."""
        return 1

    @timeit
    def _build_product_keyword_mapping(self):
        # construct product_keyword_count
        # it's actually equivalent to keyword_product_count, but can let compute
        # related_product_count faster
        self.product_keyword_count = defaultdict(dict)
        for kw, dt in self.keyword_product_count.iteritems():
            for p, c in dt.iteritems():
                self.product_keyword_count[p][kw] = c

    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            # will be used for statistics
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())

                # delegate to sub-classes
                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def round_statistics(self):
        """Get number of query, keywords, products, keyword-product relations of current round."""
        n_query = self.dbm.get_value("SELECT COUNT(*) FROM %s" % self.query_train_table)
        n_keyword = self.dbm.get_value("SELECT COUNT(*) FROM keyword")
        n_product = self.dbm.get_value("SELECT COUNT(DISTINCT product) FROM keyword_product_weight")
        n_relation = self.dbm.get_value("SELECT COUNT(*) FROM keyword_product_weight")

        print 'Round statistics: query: %d (not enough %d), keyword: %d, product: %d, relation: %d, A/M: %.2f%%' % (n_query, self._not_enough_recs, n_keyword, n_product, n_relation, 100.0*n_relation / (n_keyword*n_product))

    def recommend(self, query):
        keywords = self.ws.segment(query)
        product_weight = defaultdict(float)
        # gather product weights
        for kw in keywords:
            for product, weight in self.__fetch_related_products(kw):
                product_weight[product] += weight

        # convert dict to list for sorting
        product_weight_list = [item for item in product_weight.iteritems()]
        product_weight_list.sort(key=lambda t: t[1], reverse=True)

        if len(product_weight_list) < self.limit:
            self._not_enough_recs += 1

        return product_weight_list[:self.limit]

    def __fetch_related_products(self, keyword):
        if not self._related_product_cache.has_key(keyword):
            self._related_product_cache[keyword] = [(row['product'], row['weight']) for row in self.dbm.get_rows('SELECT product, weight FROM keyword_product_weight WHERE keyword = %s', (keyword,))]
        return self._related_product_cache[keyword]

class KeywordRecommenderHottestFallback(KeywordRecommender):
    """A recommender which uses KeywordRecommender's recommendations first,
    but turns to HottestRecommender if its recommendations are not enough."""

    def __init__(self, *args):
        """Identical to that of KeywordRecommender"""
        super(KeywordRecommenderHottestFallback, self).__init__(*args)
        self.hottest_recommender = HottestRecommender(*args)

    def __str__(self):
        return 'Keyword Recommender with Hottest Recommender fallback with %s[N=%d]' % (self.rm, self.limit)

    def preprocess(self, query_train_table):
        super(KeywordRecommenderHottestFallback, self).preprocess(query_train_table)
        self.hottest_recommender.preprocess(query_train_table)

    def recommend(self, query):
        recommendations = super(KeywordRecommenderHottestFallback, self).recommend(query)
        num_rec = len(recommendations)
        if num_rec == self.limit:
            return recommendations

        # ask HottestRecommender for more
        # note that create list in order not to break HottestRecommender.recommend_list
        hot_recommendations = self.hottest_recommender.recommend(query)[:self.limit-num_rec]

        # ensure hot_recommendations's weight is no greater than any from keyword recommendations
        max_hot_rec_weight = hot_recommendations[0][1]
        min_key_rec_weight = recommendations[-1][1] if num_rec > 0 else max_hot_rec_weight
        recommendations.extend((t[0], 1.0*min_key_rec_weight*t[1]/max_hot_rec_weight) for t in hot_recommendations)

        return recommendations

from operator import mul
def product(numbers):
    return reduce(mul, numbers)

class LinearSequenceKeywordRecommender(KeywordRecommender):
    """A tentative method using heuristic information of sequence distribution."""

    def _heuristic_weight(self, sequence):
        #return -math.log(abs(sequence-26)+1, 2)/8.0 + 1.125
        return -math.log(abs(sequence-3)+1, 2)/8.0 + 1.125

    def get_browse_count(self, sequences):
        seqs = sequences.split(',')
        #return sum(self._heuristic_weight(int(seq)) for seq in seqs)
        #return sum(self._heuristic_weight(int(seq)) for seq in seqs) * math.log(len(seqs))
        return product(self._heuristic_weight(int(seq)) for seq in seqs) * len(seqs)
        #return sum(self._heuristic_weight(int(seq)) for seq in seqs) * len(seqs)

    def __str__(self):
        return 'Linear Sequenced Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

class WeightedSequenceRelevanceMixin(object):
    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())
                # get average sequence from database
                # TODO: very inefficient, get a group all average sequences for a keyword at once
                #avg_sequence = self.dbm.get_value('select avg(sequence) from query_product where product_name = %s AND query_id in (select query_id from keyword_query where keyword = %s)', (product, keyword))
                avg_sequence = 1

                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number, avg_sequence)

                # sub-class can override sequence_weight
                # relevance *= self.sequence_weight(avg_sequence)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def sequence_weight(self, avg_sequence):
        return 1

# ensure WSRM._measure_relevance will be called with putting it before KeywordRecommender
# ref: http://python-history.blogspot.com/2010/06/method-resolution-order.html
class SequenceKeywordRecommender(WeightedSequenceRelevanceMixin, LinearSequenceKeywordRecommender):
    """This recommender weights browse count by distribution of sequence."""

    @timeit
    def preprocess(self, query_train_table):
        # first, get sequence distribution
        max_occurrence = self.dbm.get_value('SELECT MAX(c) FROM (SELECT sequence, COUNT(sequence) c FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence) T' % query_train_table)
        self.sequence_dist = {row['sequence']: float(row['ratio']) for row in self.dbm.get_rows('SELECT sequence, COUNT(sequence)/%d ratio FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence' % (max_occurrence,query_train_table))}
        self.pivot_seq = max(self.sequence_dist.iteritems(), key=lambda t:t[1])[0]

        # then, call KeywordRecommender's preprocess
        super(SequenceKeywordRecommender, self).preprocess(query_train_table)

    def _heuristic_weight(self, sequence):
        weight = self.sequence_dist[sequence]
        if self.pivot_seq-sequence < 0:
            return weight
        return 1 + weight

    def __str__(self):
        return 'Sequenced Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

class RelevanceMeasure(object):
    """Defines the RelevanceMeasure interface."""

    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        """
        @param count number of times the keyword visit the product
        @param related_product_info the tuple (related_product_number, related_product_count)
        @param related_keyword_info the tuple (related_keyword_number, related_keyword_count)
        @param all_product_number number of all products
        """
        raise NotImplemented

class BCMeasure(RelevanceMeasure):
    def get_relevance(self, count, *ignored):
        return count

    def __str__(self):
        return 'BC'

class BCIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return count * ipf

    def __str__(self):
        return 'BC-IPF'

class BFIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        bf = 1.0 * count / related_keyword_info[1]
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return bf * ipf

    def __str__(self):
        return 'BF-IPF'

if __name__ == '__main__':
    import config
    from database import DatabaseManager
    from word_segment import SpaceWordSegmenter
    dbm = DatabaseManager(config.DB_HOST, config.DB_USER, config.DB_PASSWORD, config.DB_NAME)
    try:
        word_segmenter = SpaceWordSegmenter()
        rmeasure = BCIPFMeasure()
        recommender = KeywordRecommender(10, dbm, word_segmenter, rmeasure)
        recommender.preprocess('query_train')
    finally:
        dbm.close()
<|endoftext|>


<commit_before>"""
cryptography.py
Author: Nils Kingston
Credit: Roger

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""

associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
a = ""
while a != "q":
    a = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if a == "e":
        encripting = input("What do you want to encript?")
        jumbler = input("What should the key be?")
        list(encripting)
        list(jumbler)
        jumbled = [sum(x) for x in zip(encripting, jumbler)]
        print(jumbled)

    elif a == "d":
        jumbled_code = input("Message?")
        key = input("Key?")
        g = list(jumbled_code)
        f = list(key)
        message = [x1 - x2 for (x1, x2) in zip(jumbled-code, key)]

    elif a == "q":
        print("Goodbye!")
    else:
        print("Did not understand try again")

<commit_msg>Updated from Brython Server: 10/8/2015 2:12:50 PM<commit_after>"""
cryptography.py
Author: Nils Kingston
Credit: Roger

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""

associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
a = ""
while a != "q":
    a = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if a == "e":
        encripting = input("What do you want to encript?")
        jumbler = input("What should the key be?")
        list(encripting)
        list(jumbler)
        jumbled = [sum(x) for x in zip(encripting, jumbler)]
        print(jumbled)

    elif a == "d":
        jumbled_code = input("Message?")
        key = input("Key?")
        list(jumbled_code)
        list(key)
        message = [x1 - x2 for (x1, x2) in zip(jumbled-code, key)]

    elif a == "q":
        print("Goodbye!")
    else:
        print("Did not understand try again")

<|endoftext|>


<commit_before>__author__ = 'sachinpatney'

import os, subprocess, random
repo_root = '/var/www/git/redalert'

def switchToMonaDir():
    os.chdir(repo_root + '/apis/mona')

class Mona:

    @staticmethod
    def playSound(name):
        subprocess.call(['sudo','pkill','omxplayer'])
        subprocess.call(['sudo','omxplayer',name])

    @staticmethod
    def joke():
        jokes = ['newword.mp3','policechief.mp3']
        i = round(random.random()*100) % len(jokes)
        Mona.playSound(repo_root+'/resources/sounds/'+jokes[int(i]))

    @staticmethod
    def speak(msg):
        switchToMonaDir()
        subprocess.call(['sudo', 'python', 'google.py', msg])

class IMonaJob():

    def __run__(self, time, lock):
        """Runs the job"""
        raise Exception('You must implement __run__ method on your service')<commit_msg>updating jobs<commit_after>__author__ = 'sachinpatney'

import os, subprocess, random
repo_root = '/var/www/git/redalert'

def switchToMonaDir():
    os.chdir(repo_root + '/apis/mona')

class Mona:

    @staticmethod
    def playSound(name):
        subprocess.call(['sudo','pkill','omxplayer'])
        subprocess.call(['sudo','omxplayer',name])

    @staticmethod
    def joke():
        jokes = ['newword.mp3','policechief.mp3']
        i = round(random.random()*100) % len(jokes)
        Mona.playSound(repo_root+'/resources/sounds/'+jokes[int(i)])

    @staticmethod
    def speak(msg):
        switchToMonaDir()
        subprocess.call(['sudo', 'python', 'google.py', msg])

class IMonaJob():

    def __run__(self, time, lock):
        """Runs the job"""
        raise Exception('You must implement __run__ method on your service')<|endoftext|>


<commit_before>"""
cryptography.py
Author: Payton
Credit: Avery, Daniel, Morgan

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
let=[]
kelt=[]
comb =[]
quit = False

while quit == False:
    i = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if i not in('e','d','q'):
        print("Did not understand command, try again.")
        i = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if i=="e":
        message=input("Message: ")
        key=input("Key: ")
        m=len(message)
        k=len(key)
        if m>k:
            count = key * int((m-(m%k))/k)
            trun = key[0:(m%k)]
            newkey = count + trun
        elif k>m:
            newkey = key[0:m]
            print(newkey, message)
        for x in range (0,m):
            let.append(associations.find(message[x]))
        for y in range (0,m):
            kelt.append(associations.find(newkey[y]))
        for x in range (0,m):
            if (let[x]-kelt[x])>85:
                comb.append((let[x] - kelt[x]) % 85)
        print(comb, end="")
    if i=="d":
    if i=="q":
        print("Goodbye!")
        quit = True

<commit_msg>Updated from Brython Server: 3/4/2016 2:51:21 PM<commit_after>"""
cryptography.py
Author: Payton
Credit: Avery, Daniel, Morgan

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
let=[]
kelt=[]
comb =[]
quit = False

while quit == False:
    i = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if i not in('e','d','q'):
        print("Did not understand command, try again.")
        i = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if i=="e":
        message=input("Message: ")
        key=input("Key: ")
        m=len(message)
        k=len(key)
        if m>k:
            count = key * int((m-(m%k))/k)
            trun = key[0:(m%k)]
            newkey = count + trun
        elif k>m:
            newkey = key[0:m]
            print(newkey, message)
        for x in range (0,m):
            let.append(associations.find(message[x]))
        for y in range (0,m):
            kelt.append(associations.find(newkey[y]))
        for x in range (0,m):
            if (let[x]-kelt[x])>85:
                comb.append((let[x] - kelt[x]) % 85)
        print(comb, end="")
    if i=="d":
        print
    if i=="q":
        print("Goodbye!")
        quit = True

<|endoftext|>


<commit_before>"""
cryptography.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
message= input("Please enter a message. ")
key= input("Please enter a key. ")
listnumsofmessage= []
listnumsofkey= []

for i in (message):
    listnumsofmessage.append(associations.find(i))
print (listnumsofmessage)

for i in (key):
    listnumsofkey.append(associations.find(i))
print (listnumsofkey)

print (((listnumsofmessage).count(i))%((listnumsofkey).count(i)))
    #for x in (listnumsofmessage):
        #for y in (listnumsofkey):
            #print (x,y)
        #break
    <commit_msg>Updated from Brython Server: 10/26/2015 2:55:30 PM<commit_after>"""
cryptography.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
message= input("Please enter a message. ")
key= input("Please enter a key. ")
listnumsofmessage= []
listnumsofkey= []

for i in (message):
    listnumsofmessage.append(associations.find(i))
print (listnumsofmessage)

for i in (key):
    listnumsofkey.append(associations.find(i))
print (listnumsofkey)

#print (((listnumsofmessage).count(i))%((listnumsofkey).count(i)))
    #for x in (listnumsofmessage):
        #for y in (listnumsofkey):
            #print (x,y)
        #break

for x in (listnumsofmessage):
    for y in (listnumsofkey):
        print (x, y)
    <|endoftext|>


<commit_before>"""
cryptography.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""

def cryptolist(letter2, key2):
    r = len(letter2)
    k = len(key2)
    if r > k:
        range1 = r
        listreal = letter2[:]
    if r < k:
        range1 = k
        listreal = key2[:]
    if r = k:
        range1 = k
        listreal = key2[:]
    for x in range(0, range1):
        if x > r:
            x = x - r - 1
        if x > k:
            x = x - k - 1
        valuem = letter2[x]
        valuen = key2[x]
        valueg = valuem + valuen
        listreal[x] = valueg

def efunction (i, m, k):
    letters = list(m)
    letterkey = list(k)
    r = len(letters)
    k = len(letterkey)
    letter2 = letters[:]
    key2 = letterkey[:]
    for x in range (0, r):
        p = letters[x]
        l = associations.find(p)
        letter2[x] = l
    for x in range (0, k):
        p = letterkey[x]
        l = associations.find(p)
        key2[x] = l
    cryptolist(letter2, key2)

associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
x = 0
while x == 0:
    i = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if i == "e" or i == "d":
        m = input("Message: ")
        k = input("Key: ")
        if i == "e":
            efunction (i, m, k)
        if i == "d":
            dfunction (i, m, k)
    elif i == "q":
        print ("Goodbye!")
        x = 1
    else:
        print ("Did not understand command, try again.")
        print (" ")

        <commit_msg>Updated from Brython Server: 10/5/2015 2:39:20 PM<commit_after>"""
cryptography.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""

def cryptolist(letter2, key2):
    r = len(letter2)
    k = len(key2)
    if r > k:
        range1 = r
        listreal = letter2[:]
    if r < k:
        range1 = k
        listreal = key2[:]
    if r == k:
        range1 = k
        listreal = key2[:]
    rp = False
    np = False
    z = 0
    print(letter2)
    print(key2)
    for x in range(0, range1):
        if x >= r:
            z = x%r
            np = True
        if x >= k:
            print (x)
            print(k)
            z = x%k
            print(z)
            rp = True
        if rp == True:
            valuem = letter2[x]
            valuen = key2[z]
        if np == True:
            valuem = letter2[z]
            valuen = key2[x]
        else:
            valuem = letter2[x]
            valuen = key2[x]
        valueg = valuem + valuen
        listreal[x] = valueg
    print(listreal)

def efunction (i, m, k):
    letters = list(m)
    letterkey = list(k)
    r = len(letters)
    k = len(letterkey)
    letter2 = letters[:]
    key2 = letterkey[:]
    for x in range (0, r):
        p = letters[x]
        l = associations.find(p)
        letter2[x] = l
    for x in range (0, k):
        p = letterkey[x]
        l = associations.find(p)
        key2[x] = l
    cryptolist(letter2, key2)

associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
x = 0
while x == 0:
    i = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if i == "e" or i == "d":
        m = input("Message: ")
        k = input("Key: ")
        if i == "e":
            efunction (i, m, k)
        if i == "d":
            dfunction (i, m, k)
    elif i == "q":
        print ("Goodbye!")
        x = 1
    else:
        print ("Did not understand command, try again.")
        print (" ")

        <|endoftext|>


<commit_before>"""
cryptography.py
Author: Adam Pikielny
Credit: http://www.tutorialspoint.com/python/string_len.htm

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
import sys
inp=""
while inp!="q":
    associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
    inp=input("Enter e to encrypt, d to decrypt, or q to quit: ")
    while inp!="e" and inp!="d" and inp!="q":
        print("Did not understand command, try again. \n")
        inp=input("Enter e to encrypt, d to decrypt, or q to quit: ")
    message=input("Message: ")
    key=input("Key: ")
    messagelist=[]
    for x in message:
        messagelist.append(associations.find(x))
    keylist=[]
    j=0
    while len(key)<len(message):
        key+=(key[j])
        j+=1

    for x in key:
        keylist.append(associations.find(x))

    if inp=="e":
        sumlist=[]
        for x in range(0,len(message)):
            sumlist.append(keylist[x]+messagelist[x])
        for x in sumlist:
            sys.stdout.write(associations[x%85])
            print("\n")

    if inp=="d":
        difflist=[]
        for x in range(0,len(message)):
            difflist.append(messagelist[x]-keylist[x])
        for x in difflist:
            sys.stdout.write(associations[x%85])
            print("\n")

print("Goodbye!")<commit_msg>Updated from Brython Server: October 1, 2015 5:59:25 PM EDT<commit_after>"""
cryptography.py
Author: Adam Pikielny
Credit: http://www.tutorialspoint.com/python/string_len.htm

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
import sys
inp=""
while True:
    associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
    inp=input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if inp=="q":
        break
    while inp!="e" and inp!="d" and inp!="q":
        print("Did not understand command, try again. \n")
        inp=input("Enter e to encrypt, d to decrypt, or q to quit: ")
    message=input("Message: ")
    key=input("Key: ")
    messagelist=[]
    for x in message:
        messagelist.append(associations.find(x))
    keylist=[]
    j=0
    while len(key)<len(message):
        key+=(key[j])
        j+=1

    for x in key:
        keylist.append(associations.find(x))

    if inp=="e":
        sumlist=[]
        for x in range(0,len(message)):
            sumlist.append(keylist[x]+messagelist[x])
        for x in sumlist:
            sys.stdout.write(associations[x%85])
            print("\n")

    if inp=="d":
        difflist=[]
        for x in range(0,len(message)):
            difflist.append(messagelist[x]-keylist[x])
        for x in difflist:
            sys.stdout.write(associations[x%85])
            print("\n")

print("Goodbye!")<|endoftext|>


<commit_before>from clickclick import info
from pathlib import Path
from subprocess import run
import click
import os
import platform
from .configuration import open_configuration, save_configuration
from .drive import Drive
from .paths import BASE_DIR, CONFIG_PATH, DRIVE_DIR, HOME_PATH, relative_to_home
from .scm import GitRepository, ConfigRepo

cli = click.Group()

# TODO remove support
# TODO detect dependencies
# TODO wget support

@cli.command('add-drive')
@click.argument('ACCOUNT')
@click.argument('LOCAL_NAME')
@click.argument('DRIVE_NAME')
def add_drive(account, local_name, drive_name):
    config = open_configuration()
    drive = Drive()
    drive.init(account, local_name, drive_name)
    config['drive'][local_name] = {'account': account, 'drive_name': local_name}
    save_configuration(config)
    repo = ConfigRepo()
    repo.add('ponto.yaml')
    repo.commit('Added drive from {account}/{drive_name} to ~/{local_name}'.format_map(locals()))

@cli.command('add-link')
@click.argument('TARGET')
@click.argument('LINK_NAME')
def add_link(target, link_name):
    config = open_configuration()
    link_path = relative_to_home(link_name)
    target_path = relative_to_home(target)
    info('Linking {target_path} to {link_path}'.format_map(locals()))
    link_path.expanduser().symlink_to(target_path.expanduser())
    config['ln'][str(target_path)] = str(link_path)
    save_configuration(config)
    repo = ConfigRepo()
    repo.add('ponto.yaml')
    repo.commit('Added symlink from {target_path} to {link_path}'.format_map(locals()))

@cli.command('add-repo')
@click.argument('SCM_URL')
def add_repo(scm_url):
    config = open_configuration()
    repository = GitRepository(scm_url)
    info('Cloning {repo}'.format(repo=repository))
    repository.clone()
    config['scm'].add(scm_url)
    save_configuration(config)
    repo = ConfigRepo()
    repo.add('ponto.yaml')
    repo.commit("Added '{repository}' to the repository list".format_map(locals()))

@cli.command('clone')
@click.argument("GIT_URL")
def clone(git_url: str):
    repo = ConfigRepo()
    info("Cloning Repository")
    repo.clone(git_url)

    pre_script = BASE_DIR / 'pre.sh'

    system_pre_script = BASE_DIR / 'pre-{system}.sh'.format(system=platform.system().lower())
    if system_pre_script.exists():
        info('Executing pre script for this OS')
        run(str(system_pre_script.absolute()))

    if pre_script.exists():
        info('Executing pre script')
        run(str(pre_script.absolute()))

    sync(pull=False)

@cli.command('edit-pre')
@click.option('--global', 'system', flag_value=None,
              default=True)
@click.option('--linux', 'system', flag_value='linux')
@click.option('--darwin', 'system', flag_value='darwin')
def edit_pre(system):
    # TODO edit os specific scripts
    filename = 'pre-{system}.sh'.format_map(locals()) if system else 'pre.sh'
    pre_script = BASE_DIR / filename
    run(['vim', str(pre_script.absolute())])
    pre_script.chmod(0o750)
    repo = ConfigRepo()
    repo.add('pre.sh')
    repo.commit("Updated pre script".format_map(locals()))

@cli.command('init')
@click.argument("GIT_URL")
def init(git_url):
    repo = ConfigRepo()
    # TODO receive remote and push

    info("Setting up git repository")
    git_folder = BASE_DIR / '.git'
    if not git_folder.exists():
        repo.init()
        # TODO context manager
        cwd = Path.cwd()
        os.chdir(str(BASE_DIR))
        run(['git', 'remote', 'add', 'origin', git_url])
        os.chdir(str(cwd))

    info("Creating directory structure")
    if not BASE_DIR.exists():
        BASE_DIR.mkdir()

    if not DRIVE_DIR.exists():
        DRIVE_DIR.mkdir()

    gitignore_path = BASE_DIR / '.gitignore'
    if not gitignore_path.exists():
        with gitignore_path.open('w') as gitignore:
            gitignore.write(b'drive\n')

    info("Creating config files")
    if not CONFIG_PATH.exists():
        empty_config = {'drive': dict(),
                        'scm': set(),
                        'ln': dict()}
        save_configuration(empty_config)

    cwd = Path.cwd()
    os.chdir(str(BASE_DIR))
    repo.add('ponto.yaml')
    repo.add('.gitignore')
    repo.commit('Initialization')
    repo.push()
    os.chdir(str(cwd))

@cli.command('push')
def push():
    repo = ConfigRepo()
    if HOME_PATH.exists():
        repo.add('home')
        # TODO try catch error
        repo.commit('Included local changes')
    repo.push()

@cli.command('store')
@click.argument('PATH')
def store(path):
    path = Path(path)

    if path.is_symlink():
        print("{path} is already a symlink.".format(path=path))
        return

    print("Storing {path}".format(path=path))
    relative_path = path.relative_to(Path.home())

    try:
        HOME_PATH.mkdir(parents=True)
    except FileExistsError:
        pass

    destination = HOME_PATH / relative_path

    try:
        destination.parent.mkdir(parents=True)
    except FileExistsError:
        pass

    path.rename(destination)
    path.symlink_to(destination)

    repo = ConfigRepo()
    repo.add(str('home' / relative_path))
    repo.commit("Storing {path}".format(path=path))

    # TODO error if not inside home

@cli.command('sync')
def sync(pull=True):
    if pull:
        repo = ConfigRepo()
        repo.pull()

    config = open_configuration()

    drive_points = config.get('drive', {})  # type: dict[str, dict]
    drive = Drive()
    for local_name, point in drive_points.items():
        account = point['account']
        drive_name = point['drive_name']
        # TODO ignore on error
        info('Cloning {account}/{drive_name} to ~/{local_name}'.format_map(locals()))
        drive.init(account, local_name, drive_name)

    scm_urls = config.get('scm', [])  # type: List[str]
    for url in scm_urls:
        repository = GitRepository(url)
        info('Cloning {repo}'.format(repo=repository))
        repository.clone()

    links = config.get('ln')
    for target, link in links.items():
        link_path = Path(link).expanduser()
        target_path = Path(target).expanduser()
        if not link_path.exists():
            info('Linking {target_path} to {link_path}'.format_map(locals()))
            link_path.symlink_to(target_path)
        else:
            info('{link} already exists'.format_map(locals()))

            # TODO dotfiles

    for path in HOME_PATH.glob('**/*'):
        relative_path = path.relative_to(HOME_PATH)
        link_path = Path.home() / relative_path
        if link_path.exists():
            # TODO offer to replace
            print("~/{relative_path} already exists.".format_map(locals()))
        else:
            print("Linking ~/{relative_path}.".format_map(locals()))
            link_path.symlink_to(path)
cli()
<commit_msg>Support editing commit message when editing pre script<commit_after>from clickclick import info
from pathlib import Path
from subprocess import run
import click
import os
import platform
from .configuration import open_configuration, save_configuration
from .drive import Drive
from .paths import BASE_DIR, CONFIG_PATH, DRIVE_DIR, HOME_PATH, relative_to_home
from .scm import GitRepository, ConfigRepo

cli = click.Group()

# TODO remove support
# TODO detect dependencies
# TODO wget support

@cli.command('add-drive')
@click.argument('ACCOUNT')
@click.argument('LOCAL_NAME')
@click.argument('DRIVE_NAME')
def add_drive(account, local_name, drive_name):
    config = open_configuration()
    drive = Drive()
    drive.init(account, local_name, drive_name)
    config['drive'][local_name] = {'account': account, 'drive_name': local_name}
    save_configuration(config)
    repo = ConfigRepo()
    repo.add('ponto.yaml')
    repo.commit('Added drive from {account}/{drive_name} to ~/{local_name}'.format_map(locals()))

@cli.command('add-link')
@click.argument('TARGET')
@click.argument('LINK_NAME')
def add_link(target, link_name):
    config = open_configuration()
    link_path = relative_to_home(link_name)
    target_path = relative_to_home(target)
    info('Linking {target_path} to {link_path}'.format_map(locals()))
    link_path.expanduser().symlink_to(target_path.expanduser())
    config['ln'][str(target_path)] = str(link_path)
    save_configuration(config)
    repo = ConfigRepo()
    repo.add('ponto.yaml')
    repo.commit('Added symlink from {target_path} to {link_path}'.format_map(locals()))

@cli.command('add-repo')
@click.argument('SCM_URL')
def add_repo(scm_url):
    config = open_configuration()
    repository = GitRepository(scm_url)
    info('Cloning {repo}'.format(repo=repository))
    repository.clone()
    config['scm'].add(scm_url)
    save_configuration(config)
    repo = ConfigRepo()
    repo.add('ponto.yaml')
    repo.commit("Added '{repository}' to the repository list".format_map(locals()))

@cli.command('clone')
@click.argument("GIT_URL")
def clone(git_url: str):
    repo = ConfigRepo()
    info("Cloning Repository")
    repo.clone(git_url)

    pre_script = BASE_DIR / 'pre.sh'

    system_pre_script = BASE_DIR / 'pre-{system}.sh'.format(system=platform.system().lower())
    if system_pre_script.exists():
        info('Executing pre script for this OS')
        run(str(system_pre_script.absolute()))

    if pre_script.exists():
        info('Executing pre script')
        run(str(pre_script.absolute()))

    sync(pull=False)

@cli.command('edit-pre')
@click.option('--global', 'system', flag_value=None,
              default=True)
@click.option('--linux', 'system', flag_value='linux')
@click.option('--darwin', 'system', flag_value='darwin')
def edit_pre(system):
    # TODO edit os specific scripts
    filename = 'pre-{system}.sh'.format_map(locals()) if system else 'pre.sh'
    pre_script = BASE_DIR / filename
    run(['vim', str(pre_script.absolute())])
    pre_script.chmod(0o750)
    repo = ConfigRepo()
    repo.add('pre.sh')
    commit_message = input('Commit message: ')
    commit_message = commit_message or "Updated {}".format(filename)
    repo.commit(commit_message)

@cli.command('init')
@click.argument("GIT_URL")
def init(git_url):
    repo = ConfigRepo()
    # TODO receive remote and push

    info("Setting up git repository")
    git_folder = BASE_DIR / '.git'
    if not git_folder.exists():
        repo.init()
        # TODO context manager
        cwd = Path.cwd()
        os.chdir(str(BASE_DIR))
        run(['git', 'remote', 'add', 'origin', git_url])
        os.chdir(str(cwd))

    info("Creating directory structure")
    if not BASE_DIR.exists():
        BASE_DIR.mkdir()

    if not DRIVE_DIR.exists():
        DRIVE_DIR.mkdir()

    gitignore_path = BASE_DIR / '.gitignore'
    if not gitignore_path.exists():
        with gitignore_path.open('w') as gitignore:
            gitignore.write(b'drive\n')

    info("Creating config files")
    if not CONFIG_PATH.exists():
        empty_config = {'drive': dict(),
                        'scm': set(),
                        'ln': dict()}
        save_configuration(empty_config)

    cwd = Path.cwd()
    os.chdir(str(BASE_DIR))
    repo.add('ponto.yaml')
    repo.add('.gitignore')
    repo.commit('Initialization')
    repo.push()
    os.chdir(str(cwd))

@cli.command('push')
def push():
    repo = ConfigRepo()
    if HOME_PATH.exists():
        repo.add('home')
        # TODO try catch error
        repo.commit('Included local changes')
    repo.push()

@cli.command('store')
@click.argument('PATH')
def store(path):
    path = Path(path)

    if path.is_symlink():
        print("{path} is already a symlink.".format(path=path))
        return

    print("Storing {path}".format(path=path))
    relative_path = path.relative_to(Path.home())

    try:
        HOME_PATH.mkdir(parents=True)
    except FileExistsError:
        pass

    destination = HOME_PATH / relative_path

    try:
        destination.parent.mkdir(parents=True)
    except FileExistsError:
        pass

    path.rename(destination)
    path.symlink_to(destination)

    repo = ConfigRepo()
    repo.add(str('home' / relative_path))
    repo.commit("Storing {path}".format(path=path))

    # TODO error if not inside home

@cli.command('sync')
def sync(pull=True):
    if pull:
        repo = ConfigRepo()
        repo.pull()

    config = open_configuration()

    drive_points = config.get('drive', {})  # type: dict[str, dict]
    drive = Drive()
    for local_name, point in drive_points.items():
        account = point['account']
        drive_name = point['drive_name']
        # TODO ignore on error
        info('Cloning {account}/{drive_name} to ~/{local_name}'.format_map(locals()))
        drive.init(account, local_name, drive_name)

    scm_urls = config.get('scm', [])  # type: List[str]
    for url in scm_urls:
        repository = GitRepository(url)
        info('Cloning {repo}'.format(repo=repository))
        repository.clone()

    links = config.get('ln')
    for target, link in links.items():
        link_path = Path(link).expanduser()
        target_path = Path(target).expanduser()
        if not link_path.exists():
            info('Linking {target_path} to {link_path}'.format_map(locals()))
            link_path.symlink_to(target_path)
        else:
            info('{link} already exists'.format_map(locals()))

            # TODO dotfiles

    for path in HOME_PATH.glob('**/*'):
        relative_path = path.relative_to(HOME_PATH)
        link_path = Path.home() / relative_path
        if link_path.exists():
            # TODO offer to replace
            print("~/{relative_path} already exists.".format_map(locals()))
        else:
            print("Linking ~/{relative_path}.".format_map(locals()))
            link_path.symlink_to(path)
cli()
<|endoftext|>


<commit_before>from oauthlib.oauth2.draft25 import utils
import foauth.providers

def token_uri(service, token, r):
    params = [((u'access_token', token)), ((u'api_key', service.client_id))]
    r.url = utils.add_params_to_uri(r.url, params)
    return r

class Disqus(foauth.providers.OAuth2):
    # General info about the provider
    provider_url = 'http://disqus.com/'
    docs_url = 'http://disqus.com/api/docs/'
    category = 'Social'

    # URLs to interact with the API
    authorize_url = 'https://disqus.com/api/oauth/2.0/authorize/'
    access_token_url = 'https://disqus.com/api/oauth/2.0/access_token/'
    api_domain = 'disqus.com'

    available_permissions = [
        (None, 'read data on your behalf'),
        ('write', 'write data on your behalf'),
        ('admin', 'moderate your forums'),
    ]

    bearer_type = token_uri

    def get_scope_string(self, scopes):
        # Disqus doesn't follow the spec on this point
        return ','.join(scopes)

    def get_user_id(self, key):
        r = self.api(key, self.api_domain, u'/api/3.0/users/details.json')
        return r.json[u'response'][u'id']
<commit_msg>Rewrite Disqus to use the new scope selection system<commit_after>from oauthlib.oauth2.draft25 import utils
import foauth.providers

def token_uri(service, token, r):
    params = [((u'access_token', token)), ((u'api_key', service.client_id))]
    r.url = utils.add_params_to_uri(r.url, params)
    return r

class Disqus(foauth.providers.OAuth2):
    # General info about the provider
    provider_url = 'http://disqus.com/'
    docs_url = 'http://disqus.com/api/docs/'
    category = 'Social'

    # URLs to interact with the API
    authorize_url = 'https://disqus.com/api/oauth/2.0/authorize/'
    access_token_url = 'https://disqus.com/api/oauth/2.0/access_token/'
    api_domain = 'disqus.com'

    available_permissions = [
        (None, 'read data on your behalf'),
        ('write', 'read and write data on your behalf'),
        ('admin', 'read and write data on your behalf and moderate your forums'),
    ]
    permissions_widget = 'radio'

    bearer_type = token_uri

    def get_scope_string(self, scopes):
        # Disqus doesn't follow the spec on this point
        return ','.join(scopes)

    def get_user_id(self, key):
        r = self.api(key, self.api_domain, u'/api/3.0/users/details.json')
        return r.json[u'response'][u'id']
<|endoftext|>


<commit_before>from keras import backend as K

def jaccard_distance(y_true, y_pred, smooth=100):
    """Jaccard distance is an intersection-over-union loss for semantic segmentation

    This loss is useful when you have unbalanced numbers of pixels within an image
    because it gives all classes equal weight. However, it is not the defacto
    standard for image segmentation.

    For example, assume you are trying to predict if each pixel is cat, dog, or background.
    You have 80% background pixels, 10% dog, and 10% cat. If the model predicts 100% background
    should it be be 80% right (as with categorical cross entropy) or 30% (with this loss)?

    The loss has been modified to have a smooth gradient as it converges on zero.
    This has been shifted so it converges on 0 and is smoothed to avoid exploding
    or disappearing gradient.

    Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|)
            = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|))

    # References

    Csurka, Gabriela & Larlus, Diane & Perronnin, Florent. (2013).
    What is a good evaluation measure for semantic segmentation?.
    IEEE Trans. Pattern Anal. Mach. Intell.. 26. . 10.5244/C.27.32.

    https://en.wikipedia.org/wiki/Jaccard_index

    """
    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)
    sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1)
    jac = (intersection + smooth) / (sum_ - intersection + smooth)
    return (1 - jac) * smooth
<commit_msg>jaccard.py clarify documentation<commit_after>from keras import backend as K

def jaccard_distance(y_true, y_pred, smooth=100):
    """Jaccard distance for semantic segmentation, also known as the intersection-over-union loss.

    This loss is useful when you have unbalanced numbers of pixels within an image
    because it gives all classes equal weight. However, it is not the defacto
    standard for image segmentation.

    For example, assume you are trying to predict if each pixel is cat, dog, or background.
    You have 80% background pixels, 10% dog, and 10% cat. If the model predicts 100% background
    should it be be 80% right (as with categorical cross entropy) or 30% (with this loss)?

    The loss has been modified to have a smooth gradient as it converges on zero.
    This has been shifted so it converges on 0 and is smoothed to avoid exploding
    or disappearing gradient.

    Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|)
            = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|))

    # References

    Csurka, Gabriela & Larlus, Diane & Perronnin, Florent. (2013).
    What is a good evaluation measure for semantic segmentation?.
    IEEE Trans. Pattern Anal. Mach. Intell.. 26. . 10.5244/C.27.32.

    https://en.wikipedia.org/wiki/Jaccard_index

    """
    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)
    sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1)
    jac = (intersection + smooth) / (sum_ - intersection + smooth)
    return (1 - jac) * smooth
<|endoftext|>


<commit_before>"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import keras
from .. import backend
from ..utils import anchors as utils_anchors

import numpy as np

class Anchors(keras.layers.Layer):
    def __init__(self, size, stride, ratios=None, scales=None, *args, **kwargs):
        self.size   = size
        self.stride = stride
        self.ratios = ratios
        self.scales = scales

        if ratios is None:
            self.ratios  = np.array([0.5, 1, 2], keras.backend.floatx()),
        elif isinstance(ratios, list):
            self.ratios  = np.array(ratios)
        if scales is None:
            self.scales  = np.array([2 ** 0, 2 ** (1.0 / 3.0), 2 ** (2.0 / 3.0)], keras.backend.floatx()),
        elif isinstance(scales, list):
            self.scales  = np.array(scales)

        self.num_anchors = len(ratios) * len(scales)
        self.anchors     = keras.backend.variable(utils_anchors.generate_anchors(
            base_size=size,
            ratios=ratios,
            scales=scales,
        ))

        super(Anchors, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        features = inputs
        features_shape = keras.backend.shape(features)[1:3]

        # generate proposals from bbox deltas and shifted anchors
        anchors = backend.shift(features_shape, self.stride, self.anchors)
        anchors = keras.backend.expand_dims(anchors, axis=0)

        return anchors

    def compute_output_shape(self, input_shape):
        if None not in input_shape[1:]:
            total = np.prod(input_shape[1:3]) * self.num_anchors
            return (input_shape[0], total, 4)
        else:
            return (input_shape[0], None, 4)

    def get_config(self):
        config = super(Anchors, self).get_config()
        config.update({
            'size'   : self.size,
            'stride' : self.stride,
            'ratios' : self.ratios.tolist(),
            'scales' : self.scales.tolist(),
        })

        return config

class NonMaximumSuppression(keras.layers.Layer):
    def __init__(self, nms_threshold=0.4, top_k=None, max_boxes=300, *args, **kwargs):
        self.nms_threshold = nms_threshold
        self.top_k         = top_k
        self.max_boxes     = max_boxes
        super(NonMaximumSuppression, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        boxes, classification, detections = inputs

        # TODO: support batch size > 1.
        boxes          = boxes[0]
        classification = classification[0]
        detections     = detections[0]

        scores = keras.backend.max(classification, axis=1)

        # selecting best anchors theoretically improves speed at the cost of minor performance
        if self.top_k:
            scores, indices = backend.top_k(scores, self.top_k, sorted=False)
            boxes           = keras.backend.gather(boxes, indices)
            classification  = keras.backend.gather(classification, indices)
            detections      = keras.backend.gather(detections, indices)

        indices = backend.non_max_suppression(boxes, scores, max_output_size=self.max_boxes, iou_threshold=self.nms_threshold)

        detections = keras.backend.gather(detections, indices)
        return keras.backend.expand_dims(detections, axis=0)

    def compute_output_shape(self, input_shape):
        return (input_shape[2][0], None, input_shape[2][2])

    def get_config(self):
        config = super(NonMaximumSuppression, self).get_config()
        config.update({
            'nms_threshold' : self.nms_threshold,
            'top_k'         : self.top_k,
            'max_boxes'     : self.max_boxes,
        })

        return config

class UpsampleLike(keras.layers.Layer):
    def call(self, inputs, **kwargs):
        source, target = inputs
        target_shape = keras.backend.shape(target)
        return backend.resize_images(source, (target_shape[1], target_shape[2]))

    def compute_output_shape(self, input_shape):
        return (input_shape[0][0],) + input_shape[1][1:3] + (input_shape[0][-1],)

class RegressBoxes(keras.layers.Layer):
    def __init__(self, mean=None, std=None, *args, **kwargs):
        if mean is None:
            mean = np.array([0, 0, 0, 0])
        if std is None:
            std = np.array([0.1, 0.1, 0.2, 0.2])

        if isinstance(mean, (list, tuple)):
            mean = np.array(mean)
        elif not isinstance(mean, np.ndarray):
            raise ValueError('Expected mean to be a np.ndarray, list or tuple. Received: {}'.format(type(mean)))

        if isinstance(std, (list, tuple)):
            std = np.array(std)
        elif not isinstance(std, np.ndarray):
            raise ValueError('Expected std to be a np.ndarray, list or tuple. Received: {}'.format(type(std)))

        self.mean = mean
        self.std  = std
        super(RegressBoxes, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        anchors, regression = inputs
        return backend.bbox_transform_inv(anchors, regression, mean=self.mean, std=self.std)

    def compute_output_shape(self, input_shape):
        return input_shape[0]

    def get_config(self):
        return {
            'mean': self.mean.tolist(),
            'std' : self.std.tolist(),
        }
<commit_msg>Allow batch_size > 1 for Anchors layer.<commit_after>"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import keras
from .. import backend
from ..utils import anchors as utils_anchors

import numpy as np

class Anchors(keras.layers.Layer):
    def __init__(self, size, stride, ratios=None, scales=None, *args, **kwargs):
        self.size   = size
        self.stride = stride
        self.ratios = ratios
        self.scales = scales

        if ratios is None:
            self.ratios  = np.array([0.5, 1, 2], keras.backend.floatx()),
        elif isinstance(ratios, list):
            self.ratios  = np.array(ratios)
        if scales is None:
            self.scales  = np.array([2 ** 0, 2 ** (1.0 / 3.0), 2 ** (2.0 / 3.0)], keras.backend.floatx()),
        elif isinstance(scales, list):
            self.scales  = np.array(scales)

        self.num_anchors = len(ratios) * len(scales)
        self.anchors     = keras.backend.variable(utils_anchors.generate_anchors(
            base_size=size,
            ratios=ratios,
            scales=scales,
        ))

        super(Anchors, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        features = inputs
        features_shape = keras.backend.shape(features)[:3]

        # generate proposals from bbox deltas and shifted anchors
        anchors = backend.shift(features_shape[1:3], self.stride, self.anchors)
        anchors = keras.backend.tile(keras.backend.expand_dims(anchors, axis=0), (features_shape[0], 1, 1))

        return anchors

    def compute_output_shape(self, input_shape):
        if None not in input_shape[1:]:
            total = np.prod(input_shape[1:3]) * self.num_anchors
            return (input_shape[0], total, 4)
        else:
            return (input_shape[0], None, 4)

    def get_config(self):
        config = super(Anchors, self).get_config()
        config.update({
            'size'   : self.size,
            'stride' : self.stride,
            'ratios' : self.ratios.tolist(),
            'scales' : self.scales.tolist(),
        })

        return config

class NonMaximumSuppression(keras.layers.Layer):
    def __init__(self, nms_threshold=0.4, top_k=None, max_boxes=300, *args, **kwargs):
        self.nms_threshold = nms_threshold
        self.top_k         = top_k
        self.max_boxes     = max_boxes
        super(NonMaximumSuppression, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        boxes, classification, detections = inputs

        # TODO: support batch size > 1.
        boxes          = boxes[0]
        classification = classification[0]
        detections     = detections[0]

        scores = keras.backend.max(classification, axis=1)

        # selecting best anchors theoretically improves speed at the cost of minor performance
        if self.top_k:
            scores, indices = backend.top_k(scores, self.top_k, sorted=False)
            boxes           = keras.backend.gather(boxes, indices)
            classification  = keras.backend.gather(classification, indices)
            detections      = keras.backend.gather(detections, indices)

        indices = backend.non_max_suppression(boxes, scores, max_output_size=self.max_boxes, iou_threshold=self.nms_threshold)

        detections = keras.backend.gather(detections, indices)
        return keras.backend.expand_dims(detections, axis=0)

    def compute_output_shape(self, input_shape):
        return (input_shape[2][0], None, input_shape[2][2])

    def get_config(self):
        config = super(NonMaximumSuppression, self).get_config()
        config.update({
            'nms_threshold' : self.nms_threshold,
            'top_k'         : self.top_k,
            'max_boxes'     : self.max_boxes,
        })

        return config

class UpsampleLike(keras.layers.Layer):
    def call(self, inputs, **kwargs):
        source, target = inputs
        target_shape = keras.backend.shape(target)
        return backend.resize_images(source, (target_shape[1], target_shape[2]))

    def compute_output_shape(self, input_shape):
        return (input_shape[0][0],) + input_shape[1][1:3] + (input_shape[0][-1],)

class RegressBoxes(keras.layers.Layer):
    def __init__(self, mean=None, std=None, *args, **kwargs):
        if mean is None:
            mean = np.array([0, 0, 0, 0])
        if std is None:
            std = np.array([0.1, 0.1, 0.2, 0.2])

        if isinstance(mean, (list, tuple)):
            mean = np.array(mean)
        elif not isinstance(mean, np.ndarray):
            raise ValueError('Expected mean to be a np.ndarray, list or tuple. Received: {}'.format(type(mean)))

        if isinstance(std, (list, tuple)):
            std = np.array(std)
        elif not isinstance(std, np.ndarray):
            raise ValueError('Expected std to be a np.ndarray, list or tuple. Received: {}'.format(type(std)))

        self.mean = mean
        self.std  = std
        super(RegressBoxes, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        anchors, regression = inputs
        return backend.bbox_transform_inv(anchors, regression, mean=self.mean, std=self.std)

    def compute_output_shape(self, input_shape):
        return input_shape[0]

    def get_config(self):
        return {
            'mean': self.mean.tolist(),
            'std' : self.std.tolist(),
        }
<|endoftext|>


<commit_before>import os
import copy

import anyconfig

from populus import ASSETS_DIR

from populus.utils.functional import (
    cast_return_to_tuple,
)
from populus.utils.chains import (
    get_default_ipc_path as get_geth_default_ipc_path,
)
from populus.utils.config import (
    get_nested_key,
    set_nested_key,
    pop_nested_key,
    get_empty_config,
    flatten_config_items,
    get_ini_config_file_path,
)

class empty(object):
    pass

class Config(object):
    def __init__(self, config=None, default_config_info=None):
        if config is None:
            config = get_empty_config()
        else:
            config = anyconfig.to_container(config)

        if default_config_info is None:
            default_config_info = tuple()
        self.default_config_info = default_config_info
        self.config_for_write = config
        self.config_for_read = apply_default_configs(
            self.config_for_write,
            self.default_config_info,
        )

    def get(self, key, default=None):
        try:
            return get_nested_key(self.config_for_read, key)
        except KeyError:
            return default

    def get_config(self, key, defaults=None):
        try:
            return type(self)(self[key], defaults)
        except KeyError:
            return type(self)(get_empty_config(), defaults)

    def pop(self, key, default=empty):
        try:
            value = pop_nested_key(self.config_for_read, key)
        except KeyError:
            if default is empty:
                raise
            else:
                value = default

        try:
            pop_nested_key(self.config_for_write, key)
        except KeyError:
            pass

        return value

    def setdefault(self, key, value):
        try:
            return self[key]
        except KeyError:
            self[key] = value
            return value

    @cast_return_to_tuple
    def keys(self, flatten=False):
        for key, _ in self.items(flatten=flatten):
            yield key

    @cast_return_to_tuple
    def items(self, flatten=False):
        if flatten:
            _items = flatten_config_items(self.config_for_read)
        else:
            _items = self.config_for_read.items()
        for key, value in _items:
            yield key, value

    def update(self, other, **kwargs):
        if isinstance(other, type(self)):
            self.config_for_read.update(copy.deepcopy(other.config_for_read), **kwargs)
            self.config_for_write.update(copy.deepcopy(other.config_for_write), **kwargs)
        else:
            self.config_for_read.update(copy.deepcopy(other), **kwargs)
            self.config_for_write.update(copy.deepcopy(other), **kwargs)

    def __str__(self):
        return str(self.config_for_read)

    def __repr__(self):
        return repr(self.config_for_read)

    def __eq__(self, other):
        return self.config_for_read == other

    def __bool__(self):
        if self.config_for_write:
            return True
        elif not self.default_config_info:
            return False
        else:
            return any(tuple(zip(*self.default_config_info)[1]))

    def __getitem__(self, key):
        return get_nested_key(self.config_for_read, key)

    def __setitem__(self, key, value):
        if isinstance(value, type(self)):
            set_nested_key(self.config_for_read, key, value.config_for_read)
            return set_nested_key(self.config_for_write, key, value.config_for_write)
        else:
            set_nested_key(self.config_for_read, key, value)
            return set_nested_key(self.config_for_write, key, value)

    def __contains__(self, key):
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    def __copy__(self):
        return type(self)(
            copy.copy(self.config_for_write),
            copy.copy(self.default_config_info),
        )

    def __deepcopy__(self, memo):
        return type(self)(
            copy.deepcopy(self.config_for_write, memo),
            copy.deepcopy(self.default_config_info, memo),
        )

def set_geth_mainnet_ipc_path(config):
    set_nested_key(
        config,
        'provider.settings.ipc_path',
        get_geth_default_ipc_path(testnet=False),
    )
    return config

def set_geth_ropsten_ipc_path(config):
    set_nested_key(
        config,
        'provider.settings.ipc_path',
        get_geth_default_ipc_path(testnet=True),
    )
    return config

POPULUS_CONFIG_DEFAULTS = {
    ('compilation', 'compilation.config.json', anyconfig.MS_DICTS),
    # Mainnet
    ('chains.mainnet.web3', 'chains.mainnet.web3.config.json'),
    # Ropsten
    ('chains.ropsten.web3', 'chains.ropsten.web3.config.json'),
    # TestRPC
    ('chains.testrpc.web3', 'chains.testrpc.web3.config.json'),
    # Tester
    ('chains.tester.web3', 'chains.tester.web3.config.json'),
    # Temp
    ('chains.temp.web3', 'chains.temp.web3.config.json'),
    # Web3 Configs
    ('web3.InfuraRopsten', 'web3.InfuraRopsten.config.json'),
    ('web3.InfuraMainnet', 'web3.InfuraMainnet.config.json'),
    (
        'web3.GethMainnet',
        'web3.GethMainnet.config.json',
        anyconfig.MS_NO_REPLACE,
        set_geth_mainnet_ipc_path,
    ),
    (
        'web3.GethRopsten',
        'web3.GethRopsten.config.json',
        anyconfig.MS_NO_REPLACE,
        set_geth_ropsten_ipc_path,
    ),
    (
        'web3.GethEphemeral',
        'web3.GethEphemeral.config.json',
        anyconfig.MS_NO_REPLACE,
    ),
    ('web3.TestRPC', 'web3.TestRPC.config.json'),
    ('web3.Tester', 'web3.Tester.config.json'),
}

@cast_return_to_tuple
def load_default_config_info(config_defaults=POPULUS_CONFIG_DEFAULTS):
    for config_value in config_defaults:
        if len(config_value) == 2:
            write_path, config_file_name = config_value
            merge_strategy = anyconfig.MS_NO_REPLACE
            callback = None
        elif len(config_value) == 3:
            write_path, config_file_name, merge_strategy = config_value
            callback = None
        elif len(config_value) == 4:
            write_path, config_file_name, merge_strategy, callback = config_value
        else:
            raise ValueError("Invalid Default Configuration")

        config_file_path = os.path.join(ASSETS_DIR, config_file_name)
        loaded_config = anyconfig.load(config_file_path)

        if callback is not None:
            processed_config = callback(loaded_config)
        else:
            processed_config = loaded_config

        yield write_path, processed_config, merge_strategy

def apply_default_configs(config, default_configs):
    merged_config = copy.deepcopy(config)

    for write_path, default_config, merge_strategy in default_configs:
        if write_path not in merged_config:
            set_nested_key(merged_config, write_path, default_config)
        else:
            sub_config = copy.deepcopy(merged_config[write_path])
            sub_config_with_merge_rules = anyconfig.to_container(
                sub_config,
                ac_merge=merge_strategy,
            )
            sub_config_with_merge_rules.update(default_config)

            set_nested_key(merged_config, write_path, sub_config_with_merge_rules)

    return merged_config

def load_config(config_file_path):
    config = anyconfig.load(config_file_path)
    return config

def write_config(project_dir, config, write_path):
    if write_path == get_ini_config_file_path(project_dir):
        raise ValueError(
            "The INI configuration format has been deprecated.  Please convert "
            "your configuration file to either `populus.yml` or `populus.json`"
        )

    with open(write_path, 'w') as config_file:
        anyconfig.dump(
            config,
            config_file,
            sort_keys=True,
            indent=2,
            separators=(',', ': '),
        )

    return write_path
<commit_msg>dirty<commit_after>import os
import copy

import anyconfig

from populus import ASSETS_DIR

from populus.utils.functional import (
    cast_return_to_tuple,
)
from populus.utils.chains import (
    get_default_ipc_path as get_geth_default_ipc_path,
)
from populus.utils.config import (
    get_nested_key,
    set_nested_key,
    pop_nested_key,
    get_empty_config,
    flatten_config_items,
    get_ini_config_file_path,
    resolve_config,
)

class empty(object):
    pass

class Config(object):
    parent = None
    default_config_info = None
    config_for_read = None
    config_for_write = None

    def __init__(self, config=None, default_config_info=None, parent=None):
        if config is None:
            config = get_empty_config()
        else:
            config = anyconfig.to_container(config)

        if default_config_info is None:
            default_config_info = tuple()
        self.default_config_info = default_config_info
        self.config_for_write = config
        self.config_for_read = apply_default_configs(
            self.config_for_write,
            self.default_config_info,
        )

    def get_master_config(self):
        if self.parent is None:
            return self
        else:
            return self.parent.get_master_config()

    def resolve(self, value):
        if isinstance(value, dict):
            return resolve_config(value, self.get_master_config())
        else:
            return value

    def get(self, key, default=None):
        try:
            return get_nested_key(self.config_for_read, key)
        except KeyError:
            return default

    def get_config(self, key, defaults=None):
        try:
            return type(self)(self.resolve(self[key]), defaults, parent=self)
        except KeyError:
            return type(self)(get_empty_config(), defaults, parent=self)

    def pop(self, key, default=empty):
        try:
            value = pop_nested_key(self.config_for_read, key)
        except KeyError:
            if default is empty:
                raise
            else:
                value = default

        try:
            pop_nested_key(self.config_for_write, key)
        except KeyError:
            pass

        return value

    def setdefault(self, key, value):
        try:
            return self[key]
        except KeyError:
            self[key] = value
            return value

    @cast_return_to_tuple
    def keys(self, flatten=False):
        for key, _ in self.items(flatten=flatten):
            yield key

    @cast_return_to_tuple
    def items(self, flatten=False):
        if flatten:
            _items = flatten_config_items(self.config_for_read)
        else:
            _items = self.config_for_read.items()
        for key, value in _items:
            yield key, value

    def update(self, other, **kwargs):
        if isinstance(other, type(self)):
            self.config_for_read.update(copy.deepcopy(other.config_for_read), **kwargs)
            self.config_for_write.update(copy.deepcopy(other.config_for_write), **kwargs)
        else:
            self.config_for_read.update(copy.deepcopy(other), **kwargs)
            self.config_for_write.update(copy.deepcopy(other), **kwargs)

    def __str__(self):
        return str(self.config_for_read)

    def __repr__(self):
        return repr(self.config_for_read)

    def __eq__(self, other):
        return self.config_for_read == other

    def __bool__(self):
        if self.config_for_write:
            return True
        elif not self.default_config_info:
            return False
        else:
            return any(tuple(zip(*self.default_config_info)[1]))

    def __getitem__(self, key):
        return get_nested_key(self.config_for_read, key)

    def __setitem__(self, key, value):
        if isinstance(value, type(self)):
            set_nested_key(self.config_for_read, key, value.config_for_read)
            return set_nested_key(self.config_for_write, key, value.config_for_write)
        else:
            set_nested_key(self.config_for_read, key, value)
            return set_nested_key(self.config_for_write, key, value)

    def __contains__(self, key):
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    def __iter__(self):
        return iter(self.keys())

    def __copy__(self):
        return type(self)(
            copy.copy(self.config_for_write),
            copy.copy(self.default_config_info),
        )

    def __deepcopy__(self, memo):
        return type(self)(
            copy.deepcopy(self.config_for_write, memo),
            copy.deepcopy(self.default_config_info, memo),
        )

def set_geth_mainnet_ipc_path(config):
    set_nested_key(
        config,
        'provider.settings.ipc_path',
        get_geth_default_ipc_path(testnet=False),
    )
    return config

def set_geth_ropsten_ipc_path(config):
    set_nested_key(
        config,
        'provider.settings.ipc_path',
        get_geth_default_ipc_path(testnet=True),
    )
    return config

POPULUS_CONFIG_DEFAULTS = {
    ('compilation', 'compilation.config.json', anyconfig.MS_DICTS),
    # Mainnet
    ('chains.mainnet.web3', 'chains.mainnet.web3.config.json'),
    # Ropsten
    ('chains.ropsten.web3', 'chains.ropsten.web3.config.json'),
    # TestRPC
    ('chains.testrpc.web3', 'chains.testrpc.web3.config.json'),
    # Tester
    ('chains.tester.web3', 'chains.tester.web3.config.json'),
    # Temp
    ('chains.temp.web3', 'chains.temp.web3.config.json'),
    # Web3 Configs
    ('web3.InfuraRopsten', 'web3.InfuraRopsten.config.json'),
    ('web3.InfuraMainnet', 'web3.InfuraMainnet.config.json'),
    (
        'web3.GethMainnet',
        'web3.GethMainnet.config.json',
        anyconfig.MS_NO_REPLACE,
        set_geth_mainnet_ipc_path,
    ),
    (
        'web3.GethRopsten',
        'web3.GethRopsten.config.json',
        anyconfig.MS_NO_REPLACE,
        set_geth_ropsten_ipc_path,
    ),
    (
        'web3.GethEphemeral',
        'web3.GethEphemeral.config.json',
        anyconfig.MS_NO_REPLACE,
    ),
    ('web3.TestRPC', 'web3.TestRPC.config.json'),
    ('web3.Tester', 'web3.Tester.config.json'),
}

@cast_return_to_tuple
def load_default_config_info(config_defaults=POPULUS_CONFIG_DEFAULTS):
    for config_value in config_defaults:
        if len(config_value) == 2:
            write_path, config_file_name = config_value
            merge_strategy = anyconfig.MS_NO_REPLACE
            callback = None
        elif len(config_value) == 3:
            write_path, config_file_name, merge_strategy = config_value
            callback = None
        elif len(config_value) == 4:
            write_path, config_file_name, merge_strategy, callback = config_value
        else:
            raise ValueError("Invalid Default Configuration")

        config_file_path = os.path.join(ASSETS_DIR, config_file_name)
        loaded_config = anyconfig.load(config_file_path)

        if callback is not None:
            processed_config = callback(loaded_config)
        else:
            processed_config = loaded_config

        yield write_path, processed_config, merge_strategy

def apply_default_configs(config, default_configs):
    merged_config = copy.deepcopy(config)

    for write_path, default_config, merge_strategy in default_configs:
        if write_path not in merged_config:
            set_nested_key(merged_config, write_path, default_config)
        else:
            sub_config = copy.deepcopy(merged_config[write_path])
            sub_config_with_merge_rules = anyconfig.to_container(
                sub_config,
                ac_merge=merge_strategy,
            )
            sub_config_with_merge_rules.update(default_config)

            set_nested_key(merged_config, write_path, sub_config_with_merge_rules)

    return merged_config

def load_config(config_file_path):
    config = anyconfig.load(config_file_path)
    return config

def write_config(project_dir, config, write_path):
    if write_path == get_ini_config_file_path(project_dir):
        raise ValueError(
            "The INI configuration format has been deprecated.  Please convert "
            "your configuration file to either `populus.yml` or `populus.json`"
        )

    with open(write_path, 'w') as config_file:
        anyconfig.dump(
            config,
            config_file,
            sort_keys=True,
            indent=2,
            separators=(',', ': '),
        )

    return write_path
<|endoftext|>