<commit_before>import collections

class InconsistencyError(Exception):
    pass

MineCount = collections.namedtuple('MineCount', ['total_cells', 'total_mines'])
# total_cells: total # of cells on board; all cells contained in rules + all
#   'uncharted' cells
# total_mines: total # of mines contained within all cells

class Rule(object):
    # num_mines: # of mines contained in 'cells'
    # cells: set of cell ids

    def __init__(self, num_mines, cells):
        self.num_mines = num_mines
        self.cells = set_(cells)

    def condensed(self, rule_supercells_map):
        return Rule_(self.num_mines, len(self.cells), rule_supercells_map[self])

    def __repr__(self):
        return 'Rule(num_mines=%d, cells=%s)' % (self.num_mines, sorted(list(self.cells)))

class Rule_(object):
    # num_mines: # of mines contained in 'cells_'
    # num_cells: # of base cells in 'cells_'
    # cells: set of supercells; each supercell a set of base cells

    def __init__(self, num_mines, num_cells, cells_):
        self.num_mines = num_mines
        self.num_cells = num_cells
        self.cells_ = cells_

        if self.num_mines < 0 or self.num_mines > self.num_cells:
            raise InconsistencyError()

    def decompose(self):
        if self.num_mines == 0 or self.num_mines == self.num_cells:
            for cell_ in self.cells_:
                size = len(cell_)
                yield Rule_(size if self.num_mines > 0 else 0, size, set_([cell_]))
            # degenerate rules (no cells) disappear here
        else:
            yield self

    def subtract(self, subrule):
        return Rule_(self.num_mines - subrule.num_mines,
                     self.num_cells - subrule.num_cells,
                     self.cells_ - subrule.cells_)

    def permute(self):
        pass

    def is_subrule_of(self, parent):
        return self.cells_.issubset(parent.cells_)
        # equivalent rules are subrules of each other

    def __repr__(self):
        return 'Rule_(num_mines=%d, num_cells=%d, cells_=%s)' % (self.num_mines, self.num_cells,
            sorted([sorted(list(cell_)) for cell_ in self.cells_]))

    @staticmethod
    def mk(num_mines, cells_):
        def listify(x):
            return x if hasattr(x, '__iter__') else [x]
        cells_ = [listify(cell_) for cell_ in cells_]
        return Rule_(num_mines, sum(len(cell_) for cell_ in cells_), set_(set_(cell_) for cell_ in cells_))

def solve(rules, mine_prevalence):
    # mine_prevalence is a MineCount or float (base probability that cell is mine)

    rules, _ = condense_supercells(rules)
    rules = reduce_rules(rules)

def condense_supercells(rules):
    cell_rules_map = map_reduce(rules, lambda rule: [(cell, rule) for cell in rule.cells], set_)
    rules_supercell_map = map_reduce(cell_rules_map.iteritems(), lambda (cell, rules): [(rules, cell)], set_)
    rule_supercells_map = map_reduce(rules_supercell_map.iteritems(), lambda (rules, cell_): [(rule, cell_) for rule in rules], set_)
    return ([rule.condensed(rule_supercells_map) for rule in rules], rules_supercell_map.values())

def reduce_rules(rules):
    rr = RuleReducer()
    rr.add_rules(rules)
    return rr.reduce_all()

class Reduceable(object):
    def __init__(self, superrule, subrule):
        self.superrule = superrule
        self.subrule = subrule

    def metric(self):
        num_reduced_cells = self.superrule.num_cells - self.subrule.num_cells
        num_reduced_mines = self.superrule.num_mines - self.subrule.num_mines
        # favor reductions that involve bigger rules, and amongst same-sized rules, those
        # that yield # mines towards the extremes -- such rules have fewer permutations
        return (self.superrule.num_cells, self.subrule.num_cells,
                abs(num_reduced_mines - .5 * num_reduced_cells))

    def reduce(self):
        return self.superrule.subtract(self.subrule)

    def contains(self, rule):
        return rule in (self.superrule, self.subrule)

class RuleReducer(object):
    def __init__(self):
        # current list of rules
        self.active_rules = []
        # mapping of cells to list of rules containing that cell
        self.cell_rules_map = collections.defaultdict(list)
        # current list of all possible reductions
        self.candidate_reductions = [] #could make this a priority queue for efficiency

    def add_rules(self, rules):
        for rule in rules:
            self.add_rule(rule)

    def add_rule(self, rule):
        for base_rule in rule.decompose():
            self.add_base_rule(base_rule)

    def add_base_rule(self, rule):
        self.active_rules.append(rule)
        # update reduceables before cell index or else rule will reduce against itself
        self.update_reduceables(rule)
        for cell_ in rule.cells_:
            self.cell_rules_map[cell_].append(rule)

    def update_reduceables(self, rule):
        overlapping_rules = reduce(lambda a, b: a.union(b), (self.cell_rules_map[cell_] for cell_ in rule.cells_), set())
        for rule_ov in overlapping_rules:
            if rule_ov.is_subrule_of(rule):
                # catches if rules are equivalent
                self.candidate_reductions.append(Reduceable(rule, rule_ov))
            elif rule.is_subrule_of(rule_ov):
                self.candidate_reductions.append(Reduceable(rule_ov, rule))

    def remove_rule(self, rule):
        self.active_rules.remove(rule)
        for cell_ in rule.cells_:
            self.cell_rules_map[cell_].remove(rule)
        # could make this more efficient with an index rule -> reduceables
        for reduc in self.candidate_reductions:
            if reduc.contains(rule):
                self.candidate_reductions.remove(reduc)

    def pop_best_reduction(self):
        reduction = max(self.candidate_reductions, key=lambda reduc: reduc.metric())
        self.candidate_reductions.remove(reduction)
        return reduction

    def reduce(self, reduction):
        reduced_rule = reduction.reduce()
        self.remove_rule(reduction.superrule)
        self.add_rule(reduced_rule)

    def reduce_all(self):
        while self.candidate_reductions:
            self.reduce(self.pop_best_reduction())

        return self.active_rules

#  fun reduceRules (candidateRules : rule IntMap.map, allCells : IntRel) (*: (rule IntMap.map * graph)*) = let
#    fun addRule (candidateRule, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#      fun add (rule, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#        val ruleid = getUid()
#        val {cells, ...} = rule
#        val numCells = IntSet.numItems(cells)
#        val ruleFull = (ruleid, rule)
#
#        fun indexRuleByCell (cell, (cellRulesMap, ruleInterference, ruleReductions)) = let
#          val relatedRules = case IntMap.find(cellRulesMap, cell) of
#                               SOME x => x
#                             | NONE => IntSet.empty
#
#          fun relateRule () = let
#            fun relateRuleByCell (relatedRule, (ruleInterference, ruleReductions)) = let
#              val otherRule = valOf(IntMap.find(builtRules, relatedRule))
#              val numCellsOther = IntSet.numItems(#cells otherRule)
#              val otherRuleFull = (relatedRule, otherRule)
#
#              val subset = IntSet.add(case graphEdgeData(ruleInterference, (ruleid, relatedRule)) of
#                                        SOME x => x
#                                      | NONE => IntSet.empty, cell)
#
#              val reduction = if IntSet.numItems subset = numCells then
#                                SOME {subset = ruleFull, superset = otherRuleFull}
#                              else if IntSet.numItems subset = numCellsOther then
#                                SOME {subset = otherRuleFull, superset = ruleFull}
#                              else
#                                NONE
#            in
#              (graphAddEdge(ruleInterference, (ruleid, relatedRule), subset),
#               case reduction of SOME x => RRPQ.insert(x, ruleReductions)
#                               | NONE => ruleReductions)
#            end
#          in
#            IntSet.foldl relateRuleByCell (ruleInterference, ruleReductions) relatedRules
#          end
#
#          val cellRulesMap = IntMap.insert(cellRulesMap, cell, IntSet.add(relatedRules, ruleid))
#          val (ruleInterference, ruleReductions) = relateRule()
#        in
#          (cellRulesMap, ruleInterference, ruleReductions)
#        end
#
#        val builtRules = IntMap.insert(builtRules, ruleid, rule)
#        val ruleInterference = graphAddNode(ruleInterference, ruleid)
#        val (cellRulesMap, ruleInterference, ruleReductions) = IntSet.foldl indexRuleByCell (cellRulesMap, ruleInterference, ruleReductions) cells
#      in
#        (* debug *) print ("adding rule " ^ (Int.toString ruleid) ^ " (" ^ (strrule rule) ^ "):\n");
#        (* debug *) print ("rule added. current state:\n" ^ strreducstate(builtRules, cellRulesMap, ruleInterference, ruleReductions));
#
#        (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#      end
#
#      fun degenerateRule (cell, hasMines) = let
#        val numCells' = IntSet.numItems(valOf(IntMap.find(allCells, cell)))
#        val numMines = if hasMines then numCells' else 0
#      in
#        {numCells' = numCells', numMines = numMines, cells = IntSet.singleton(cell)}
#      end
#
#      val {numMines, numCells', cells} = candidateRule
#      val numCells = IntSet.numItems cells
#    in
#      (* debug *) print ("adding candidate (" ^ (strrule candidateRule) ^ "):\n");
#
#      if (numMines < 0) orelse (numMines > numCells') then (* inconsistent *)
#        ((* debug *) print "inconsistent rule!!\n";
#        raise Inconsistent
#        )
#      else if numCells = 0 then (* empty *)
#        ((* debug *) print "empty rule\n";
#        (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#        )
#      else if numCells > 1 andalso ((numMines = 0) orelse (numMines = numCells')) then (* splittable *)
#        ((* debug *) print "splitting rule\n";
#        IntSet.foldl (fn (cell, indexState) => add(degenerateRule(cell, numMines > 0), indexState)) (builtRules, cellRulesMap, ruleInterference, ruleReductions) cells
#        )
#      else
#        add(candidateRule, (builtRules, cellRulesMap, ruleInterference, ruleReductions))
#    end
#
#    fun removeRule (ruleid, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#      val (builtRules, {cells, ...} : rule) = IntMap.remove(builtRules, ruleid)
#      val cellRulesMap = IntSet.foldl (fn (cell, crm) =>  IntMap.insert(crm, cell, IntSet.delete(valOf(IntMap.find(crm, cell)), ruleid))) cellRulesMap cells
#      val ruleInterference = graphDeleteNode(ruleInterference, ruleid)
#      val ruleReductions = RRPQfilter (fn ({subset = (sub, _), superset = (sup, _)}) => not (sub = ruleid orelse sup = ruleid)) ruleReductions
#    in
#      (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#    end
#
#    fun subtractRule ({numCells' = subNumCells', numMines = subNumMines, cells = subCells},
#                      {numCells' = supNumCells', numMines = supNumMines, cells = supCells}) =
#      {numCells' = supNumCells' - subNumCells', numMines = supNumMines - subNumMines, cells = IntSet.difference(supCells, subCells)}
#
#    fun reduceRule (reduction, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#      val {subset = (_, subsetRule), superset = (superset, supersetRule)} = reduction
#
#      (* debug *) val xxx = let in print ("reducing rule " ^ (Int.toString superset) ^ " by rule " ^ (Int.toString (#1 (#subset reduction))) ^ "\n") end
#
#      val reducedRule = subtractRule(subsetRule, supersetRule)
#      val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = removeRule(superset, (builtRules, cellRulesMap, ruleInterference, ruleReductions))
#      val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = addRule(reducedRule, (builtRules, cellRulesMap, ruleInterference, ruleReductions))
#    in
#      (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#    end
#
#    fun reduceAll (builtRules, cellRulesMap, ruleInterference, ruleReductions) = if RRPQ.isEmpty ruleReductions then
#      (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#    else let val (reduction, ruleReductions) = RRPQ.remove(ruleReductions) in
#      reduceAll(reduceRule(reduction, (builtRules, cellRulesMap, ruleInterference, ruleReductions)))
#    end
#
#    val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = IntMap.foldl addRule (IntMap.empty, IntMap.empty, graphEmpty(), RRPQ.empty) candidateRules
#    val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = reduceAll(builtRules, cellRulesMap, ruleInterference, ruleReductions)
#  in
#    (* debug *) print ("\n\nfinal reduction:\n" ^ strreducstate(builtRules, cellRulesMap, ruleInterference, ruleReductions));
#    (builtRules, cellRulesMap, ruleInterference) (* (builtRules, ruleInterference) *)
#  end

set_ = frozenset

def map_reduce(data, emitfunc=lambda rec: [(rec,)], reducefunc=lambda v: v):
    """perform a "map-reduce" on the data

    emitfunc(datum): return an iterable of key-value pairings as (key, value). alternatively, may
        simply emit (key,) (useful for reducefunc=len)
    reducefunc(values): applied to each list of values with the same key; defaults to just
        returning the list
    data: iterable of data to operate on
    """
    mapped = collections.defaultdict(list)
    for rec in data:
        for emission in emitfunc(rec):
            try:
                k, v = emission
            except ValueError:
                k, v = emission[0], None
            mapped[k].append(v)
    return dict((k, reducefunc(v)) for k, v in mapped.iteritems())
<commit_msg>fix bugs regarding deleting from a list being iterated upon<commit_after>import collections

class InconsistencyError(Exception):
    pass

MineCount = collections.namedtuple('MineCount', ['total_cells', 'total_mines'])
# total_cells: total # of cells on board; all cells contained in rules + all
#   'uncharted' cells
# total_mines: total # of mines contained within all cells

class Rule(object):
    # num_mines: # of mines contained in 'cells'
    # cells: set of cell ids

    def __init__(self, num_mines, cells):
        self.num_mines = num_mines
        self.cells = set_(cells)

    def condensed(self, rule_supercells_map):
        return Rule_(self.num_mines, len(self.cells), rule_supercells_map[self])

    def __repr__(self):
        return 'Rule(num_mines=%d, cells=%s)' % (self.num_mines, sorted(list(self.cells)))

class Rule_(object):
    # num_mines: # of mines contained in 'cells_'
    # num_cells: # of base cells in 'cells_'
    # cells: set of supercells; each supercell a set of base cells

    def __init__(self, num_mines, num_cells, cells_):
        self.num_mines = num_mines
        self.num_cells = num_cells
        self.cells_ = cells_

        if self.num_mines < 0 or self.num_mines > self.num_cells:
            raise InconsistencyError()

    def decompose(self):
        if self.num_mines == 0 or self.num_mines == self.num_cells:
            for cell_ in self.cells_:
                size = len(cell_)
                yield Rule_(size if self.num_mines > 0 else 0, size, set_([cell_]))
            # degenerate rules (no cells) disappear here
        else:
            yield self

    def subtract(self, subrule):
        return Rule_(self.num_mines - subrule.num_mines,
                     self.num_cells - subrule.num_cells,
                     self.cells_ - subrule.cells_)

    def permute(self):
        pass

    def is_subrule_of(self, parent):
        return self.cells_.issubset(parent.cells_)
        # equivalent rules are subrules of each other

    def __repr__(self):
        return 'Rule_(num_mines=%d, num_cells=%d, cells_=%s)' % (self.num_mines, self.num_cells,
            sorted([sorted(list(cell_)) for cell_ in self.cells_]))

    @staticmethod
    def mk(num_mines, cells_):
        def listify(x):
            return x if hasattr(x, '__iter__') else [x]
        cells_ = [listify(cell_) for cell_ in cells_]
        return Rule_(num_mines, sum(len(cell_) for cell_ in cells_), set_(set_(cell_) for cell_ in cells_))

def solve(rules, mine_prevalence):
    # mine_prevalence is a MineCount or float (base probability that cell is mine)

    rules, _ = condense_supercells(rules)
    rules = reduce_rules(rules)

def condense_supercells(rules):
    cell_rules_map = map_reduce(rules, lambda rule: [(cell, rule) for cell in rule.cells], set_)
    rules_supercell_map = map_reduce(cell_rules_map.iteritems(), lambda (cell, rules): [(rules, cell)], set_)
    rule_supercells_map = map_reduce(rules_supercell_map.iteritems(), lambda (rules, cell_): [(rule, cell_) for rule in rules], set_)
    return ([rule.condensed(rule_supercells_map) for rule in rules], rules_supercell_map.values())

def reduce_rules(rules):
    rr = RuleReducer()
    rr.add_rules(rules)
    return rr.reduce_all()

class Reduceable(object):
    def __init__(self, superrule, subrule):
        self.superrule = superrule
        self.subrule = subrule

    def metric(self):
        num_reduced_cells = self.superrule.num_cells - self.subrule.num_cells
        num_reduced_mines = self.superrule.num_mines - self.subrule.num_mines
        # favor reductions that involve bigger rules, and amongst same-sized rules, those
        # that yield # mines towards the extremes -- such rules have fewer permutations
        return (self.superrule.num_cells, self.subrule.num_cells,
                abs(num_reduced_mines - .5 * num_reduced_cells))

    def reduce(self):
        return self.superrule.subtract(self.subrule)

    def contains(self, rule):
        return rule in (self.superrule, self.subrule)

    def __repr__(self):
        return 'Reduceable(superrule=%s, subrule=%s)' % (self.superrule, self.subrule)

class RuleReducer(object):
    def __init__(self):
        # current list of rules
        self.active_rules = set()
        # mapping of cells to list of rules containing that cell
        self.cell_rules_map = collections.defaultdict(set)
        # current list of all possible reductions
        self.candidate_reductions = set() #could make this a priority queue for efficiency

    def add_rules(self, rules):
        for rule in rules:
            self.add_rule(rule)

    def add_rule(self, rule):
        for base_rule in rule.decompose():
            self.add_base_rule(base_rule)

    def add_base_rule(self, rule):
        self.active_rules.add(rule)
        # update reduceables before cell index or else rule will reduce against itself
        self.update_reduceables(rule)
        for cell_ in rule.cells_:
            self.cell_rules_map[cell_].add(rule)

    def update_reduceables(self, rule):
        overlapping_rules = reduce(lambda a, b: a.union(b), (self.cell_rules_map[cell_] for cell_ in rule.cells_), set())
        for rule_ov in overlapping_rules:
            if rule_ov.is_subrule_of(rule):
                # catches if rules are equivalent
                self.candidate_reductions.add(Reduceable(rule, rule_ov))
            elif rule.is_subrule_of(rule_ov):
                self.candidate_reductions.add(Reduceable(rule_ov, rule))

    def remove_rule(self, rule):
        self.active_rules.remove(rule)
        for cell_ in rule.cells_:
            self.cell_rules_map[cell_].remove(rule)
        # could make this more efficient with an index rule -> reduceables
        self.candidate_reductions = set(reduc for reduc in self.candidate_reductions if not reduc.contains(rule))

    def pop_best_reduction(self):
        reduction = max(self.candidate_reductions, key=lambda reduc: reduc.metric())
        self.candidate_reductions.remove(reduction)
        return reduction

    def reduce(self, reduction):
        reduced_rule = reduction.reduce()
        self.remove_rule(reduction.superrule)
        self.add_rule(reduced_rule)

    def reduce_all(self):
        while self.candidate_reductions:
            self.reduce(self.pop_best_reduction())

        return self.active_rules

#  fun reduceRules (candidateRules : rule IntMap.map, allCells : IntRel) (*: (rule IntMap.map * graph)*) = let
#    fun addRule (candidateRule, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#      fun add (rule, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#        val ruleid = getUid()
#        val {cells, ...} = rule
#        val numCells = IntSet.numItems(cells)
#        val ruleFull = (ruleid, rule)
#
#        fun indexRuleByCell (cell, (cellRulesMap, ruleInterference, ruleReductions)) = let
#          val relatedRules = case IntMap.find(cellRulesMap, cell) of
#                               SOME x => x
#                             | NONE => IntSet.empty
#
#          fun relateRule () = let
#            fun relateRuleByCell (relatedRule, (ruleInterference, ruleReductions)) = let
#              val otherRule = valOf(IntMap.find(builtRules, relatedRule))
#              val numCellsOther = IntSet.numItems(#cells otherRule)
#              val otherRuleFull = (relatedRule, otherRule)
#
#              val subset = IntSet.add(case graphEdgeData(ruleInterference, (ruleid, relatedRule)) of
#                                        SOME x => x
#                                      | NONE => IntSet.empty, cell)
#
#              val reduction = if IntSet.numItems subset = numCells then
#                                SOME {subset = ruleFull, superset = otherRuleFull}
#                              else if IntSet.numItems subset = numCellsOther then
#                                SOME {subset = otherRuleFull, superset = ruleFull}
#                              else
#                                NONE
#            in
#              (graphAddEdge(ruleInterference, (ruleid, relatedRule), subset),
#               case reduction of SOME x => RRPQ.insert(x, ruleReductions)
#                               | NONE => ruleReductions)
#            end
#          in
#            IntSet.foldl relateRuleByCell (ruleInterference, ruleReductions) relatedRules
#          end
#
#          val cellRulesMap = IntMap.insert(cellRulesMap, cell, IntSet.add(relatedRules, ruleid))
#          val (ruleInterference, ruleReductions) = relateRule()
#        in
#          (cellRulesMap, ruleInterference, ruleReductions)
#        end
#
#        val builtRules = IntMap.insert(builtRules, ruleid, rule)
#        val ruleInterference = graphAddNode(ruleInterference, ruleid)
#        val (cellRulesMap, ruleInterference, ruleReductions) = IntSet.foldl indexRuleByCell (cellRulesMap, ruleInterference, ruleReductions) cells
#      in
#        (* debug *) print ("adding rule " ^ (Int.toString ruleid) ^ " (" ^ (strrule rule) ^ "):\n");
#        (* debug *) print ("rule added. current state:\n" ^ strreducstate(builtRules, cellRulesMap, ruleInterference, ruleReductions));
#
#        (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#      end
#
#      fun degenerateRule (cell, hasMines) = let
#        val numCells' = IntSet.numItems(valOf(IntMap.find(allCells, cell)))
#        val numMines = if hasMines then numCells' else 0
#      in
#        {numCells' = numCells', numMines = numMines, cells = IntSet.singleton(cell)}
#      end
#
#      val {numMines, numCells', cells} = candidateRule
#      val numCells = IntSet.numItems cells
#    in
#      (* debug *) print ("adding candidate (" ^ (strrule candidateRule) ^ "):\n");
#
#      if (numMines < 0) orelse (numMines > numCells') then (* inconsistent *)
#        ((* debug *) print "inconsistent rule!!\n";
#        raise Inconsistent
#        )
#      else if numCells = 0 then (* empty *)
#        ((* debug *) print "empty rule\n";
#        (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#        )
#      else if numCells > 1 andalso ((numMines = 0) orelse (numMines = numCells')) then (* splittable *)
#        ((* debug *) print "splitting rule\n";
#        IntSet.foldl (fn (cell, indexState) => add(degenerateRule(cell, numMines > 0), indexState)) (builtRules, cellRulesMap, ruleInterference, ruleReductions) cells
#        )
#      else
#        add(candidateRule, (builtRules, cellRulesMap, ruleInterference, ruleReductions))
#    end
#
#    fun removeRule (ruleid, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#      val (builtRules, {cells, ...} : rule) = IntMap.remove(builtRules, ruleid)
#      val cellRulesMap = IntSet.foldl (fn (cell, crm) =>  IntMap.insert(crm, cell, IntSet.delete(valOf(IntMap.find(crm, cell)), ruleid))) cellRulesMap cells
#      val ruleInterference = graphDeleteNode(ruleInterference, ruleid)
#      val ruleReductions = RRPQfilter (fn ({subset = (sub, _), superset = (sup, _)}) => not (sub = ruleid orelse sup = ruleid)) ruleReductions
#    in
#      (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#    end
#
#    fun subtractRule ({numCells' = subNumCells', numMines = subNumMines, cells = subCells},
#                      {numCells' = supNumCells', numMines = supNumMines, cells = supCells}) =
#      {numCells' = supNumCells' - subNumCells', numMines = supNumMines - subNumMines, cells = IntSet.difference(supCells, subCells)}
#
#    fun reduceRule (reduction, (builtRules, cellRulesMap, ruleInterference, ruleReductions)) = let
#      val {subset = (_, subsetRule), superset = (superset, supersetRule)} = reduction
#
#      (* debug *) val xxx = let in print ("reducing rule " ^ (Int.toString superset) ^ " by rule " ^ (Int.toString (#1 (#subset reduction))) ^ "\n") end
#
#      val reducedRule = subtractRule(subsetRule, supersetRule)
#      val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = removeRule(superset, (builtRules, cellRulesMap, ruleInterference, ruleReductions))
#      val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = addRule(reducedRule, (builtRules, cellRulesMap, ruleInterference, ruleReductions))
#    in
#      (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#    end
#
#    fun reduceAll (builtRules, cellRulesMap, ruleInterference, ruleReductions) = if RRPQ.isEmpty ruleReductions then
#      (builtRules, cellRulesMap, ruleInterference, ruleReductions)
#    else let val (reduction, ruleReductions) = RRPQ.remove(ruleReductions) in
#      reduceAll(reduceRule(reduction, (builtRules, cellRulesMap, ruleInterference, ruleReductions)))
#    end
#
#    val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = IntMap.foldl addRule (IntMap.empty, IntMap.empty, graphEmpty(), RRPQ.empty) candidateRules
#    val (builtRules, cellRulesMap, ruleInterference, ruleReductions) = reduceAll(builtRules, cellRulesMap, ruleInterference, ruleReductions)
#  in
#    (* debug *) print ("\n\nfinal reduction:\n" ^ strreducstate(builtRules, cellRulesMap, ruleInterference, ruleReductions));
#    (builtRules, cellRulesMap, ruleInterference) (* (builtRules, ruleInterference) *)
#  end

set_ = frozenset

def map_reduce(data, emitfunc=lambda rec: [(rec,)], reducefunc=lambda v: v):
    """perform a "map-reduce" on the data

    emitfunc(datum): return an iterable of key-value pairings as (key, value). alternatively, may
        simply emit (key,) (useful for reducefunc=len)
    reducefunc(values): applied to each list of values with the same key; defaults to just
        returning the list
    data: iterable of data to operate on
    """
    mapped = collections.defaultdict(list)
    for rec in data:
        for emission in emitfunc(rec):
            try:
                k, v = emission
            except ValueError:
                k, v = emission[0], None
            mapped[k].append(v)
    return dict((k, reducefunc(v)) for k, v in mapped.iteritems())
<|endoftext|>


<commit_before>from constants import *
import sys, time, datetime
from scraper import Scraper
from dateutil import tz
from execute_query import execute_query
from writer import log

start_time = time.time()
keep_going = True
while keep_going:
    team_ids = [str(i[0]) for i in execute_query('SELECT team_id FROM pending_changes_wcl LIMIT {0}'.format(MAX_ALLOCATED))]
    if not team_ids:
        log('info','No pending teams. Sleeping for one minute.')
        time.sleep(60)
    else:
        teams = Scraper('warcraftlogs',team_ids,start_time,'tornado')
        keep_going = teams.check_warcraftlogs(True)
        execute_query('DELETE FROM pending_changes_wcl WHERE team_id IN ({0})'.format(','.join(team_ids)))
    if time.time() - start_time > MAXIMUM_RUNTIME: keep_going = False

log('info','{0}. Aborting now.'.format('Reached the maximum runtime' if (time.time() - start_time >= MAXIMUM_RUNTIME) else 'Done'))

<commit_msg>Add some log info<commit_after>from constants import *
import sys, time, datetime
from scraper import Scraper
from dateutil import tz
from execute_query import execute_query
from writer import log

start_time = time.time()
keep_going = True
while keep_going:
    team_ids = [str(i[0]) for i in execute_query('SELECT team_id FROM pending_changes_wcl LIMIT {0}'.format(MAX_ALLOCATED))]
    if not team_ids:
        log('info','No pending teams. Sleeping for one minute.')
        time.sleep(60)
    else:
        log('info','Refreshing {0} pending teams.'.format(len(team_ids)))
        teams = Scraper('warcraftlogs',team_ids,start_time,'tornado')
        keep_going = teams.check_warcraftlogs(True)
        execute_query('DELETE FROM pending_changes_wcl WHERE team_id IN ({0})'.format(','.join(team_ids)))
    if time.time() - start_time > MAXIMUM_RUNTIME: keep_going = False

log('info','{0}. Aborting now.'.format('Reached the maximum runtime' if (time.time() - start_time >= MAXIMUM_RUNTIME) else 'Done'))

<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function, division

import argparse
from collections import defaultdict
import datetime
import json
import psutil
import os
import shutil
import subprocess
import time
import networkx as nx
import sys

from conda_build.metadata import parse, MetaData

CONDA_BUILD_CACHE=os.environ.get("CONDA_BUILD_CACHE")

class PopenWrapper(object):
    # Small wrapper around subprocess.Popen to allow memory usage monitoring

    def __init__(self, *args, **kwargs):
        self.elapsed = None
        self.rss = None
        self.vms = None
        # set returncode to a bad one
        # in case it is never defined
        # after here.
        self.returncode = 173
        self.disk = None

        self._execute(*args, **kwargs)

    def _execute(self, *args, **kwargs):
        # The polling interval (in seconds)
        time_int = kwargs.pop('time_int', 1)

        # Create a process of this (the parent) process
        parent = psutil.Process(os.getpid())
        initial_usage = psutil.disk_usage(sys.prefix).used

        # Using the convenience Popen class provided by psutil
        start_time = time.time()
        _popen = psutil.Popen(*args, **kwargs)
        try:
            while _popen.is_running():
                #We need to get all of the children of our process since our process spawns other processes
                # Collect all of the child processes

                try:
                    # We use the parent process to get mem usage of all spawned processes
                    child_pids = [_.memory_info() for _ in parent.children(recursive=True) if _.is_running()]
                    # Sum the memory usage of all the children together (2D columnwise sum)
                    rss, vms = [sum(_) for _ in zip(*child_pids)]

                    self.rss = max(rss, self.rss)
                    self.vms = max(vms, self.vms)

                    # Get disk usage
                    used_disk = initial_usage - psutil.disk_usage(sys.prefix).used
                    self.disk = max(used_disk, self.disk)

                except psutil.AccessDenied as e:
                    if _popen.status() == psutil.STATUS_ZOMBIE:
                        _popen.wait()

                time.sleep(time_int)
                self.elapsed = time.time() - start_time
                self.returncode = _popen.returncode
                if _popen.returncode is not None and _popen.is_running():
                    _popen.kill()
                    break
        except KeyboardInterrupt:
            _popen.kill()
            raise

    def __repr__(self):
        return str({'elapsed': self.elapsed,
                    'rss': self.rss,
                    'vms': self.vms,
                    'returncode': self.returncode})

def bytes2human(n):
    # http://code.activestate.com/recipes/578019
    # >>> bytes2human(10000)
    # '9.8K'
    # >>> bytes2human(100001221)
    # '95.4M'
    symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
    prefix = {}
    for i, s in enumerate(symbols):
        prefix[s] = 1 << (i + 1) * 10
    for s in reversed(symbols):
        if n >= prefix[s]:
            value = float(n) / prefix[s]
            return '%.1f%s' % (value, s)
    return "%sB" % n

def last_changed_git_branch(git_root):
    args = ['git', 'for-each-ref',
            '--sort=-committerdate', 'refs/heads/',]
    proc = subprocess.Popen(args,
                            cwd=git_root,
                            stdout=subprocess.PIPE)
    if proc.wait():
        raise ValueError('Bad return code '
                         'from git branch sort', proc.poll())
    head_1 = proc.stdout.read().decode().splitlines()[0]
    branch = head_1.split()[-1]
    print('Last changed branch: ', branch)
    return branch

def git_changed_files(git_rev, git_root=''):
    """
    Get the list of files changed in a git revision and return a list of package directories that have been modified.
    """
    proc = subprocess.Popen(['git', 'diff-tree',
                              '--no-commit-id', '--name-only',
                              '-r', git_rev
                              ],
                              cwd=git_root,
                              stdout=subprocess.PIPE)
    if proc.wait():
        raise ValueError('Bad git return code: {}'.format(proc.poll()))
    files = proc.stdout.read().decode().splitlines()
    too_short = ('\\','/')
    changed = {os.path.dirname(f) for f in files}
    changed = {f for f in changed if f and f not in too_short}
    return changed

def read_recipe(path):
    return MetaData(path)

def describe_meta(meta):
    """Return a dictionary that describes build info of meta.yaml"""

    # Things we care about and need fast access to:
    #   1. Package name and version
    #   2. Build requirements
    #   3. Build number
    #   4. Recipe directory
    d = {}

    d['build'] = meta.get_value('build/number', 0)
    d['depends'] = format_deps(meta.get_value('requirements/build'))
    d['version'] = meta.get_value('package/version')
    return d

def format_deps(deps):
    d = {}
    for x in deps:
        x = x.strip().split()
        if len(x) == 2:
            d[x[0]] = x[1]
        else:
            d[x[0]] = ''
    return d

def get_build_deps(recipe):
    return format_deps(recipe.get_value('requirements/build'))

def construct_graph(directory, filter_by_git_change=True):
    '''
    Construct a directed graph of dependencies from a directory of recipes

    Annotate dependencies that don't have recipes in that directory
    '''
    print('construct_graph with args: ', directory, filter_by_git_change)
    g = nx.DiGraph()
    build_numbers = {}
    directory = os.path.abspath(directory)
    assert os.path.isdir(directory)

    # get all immediate subdirectories
    recipe_dirs = next(os.walk(directory))[1]
    recipe_dirs = set(x for x in recipe_dirs if not x.startswith('.'))
    if filter_by_git_change:
        changed_recipes = git_changed_files('HEAD', git_root=directory)
        print('changed_recipes {}'.format(changed_recipes))
    for rd in recipe_dirs:
        recipe_dir = os.path.join(directory, rd)
        try:
            pkg = read_recipe(recipe_dir)
            name = pkg.name()
        except:
            continue

        # add package (in case it has no build deps)
        if filter_by_git_change:
            _dirty = True if rd in changed_recipes else False
        else:
            _dirty = True
        g.add_node(name, meta=describe_meta(pkg), recipe=recipe_dir, dirty=_dirty)
        for k, d in get_build_deps(pkg).iteritems():
            g.add_edge(name, k)
    return g

def dirty(graph, implicit=True):
    """
    Return a set of all dirty nodes in the graph.

    These include implicit and explicit dirty nodes.
    """
    # Reverse the edges to get true dependency
    dirty_nodes = {n for n, v in graph.node.items() if v.get('dirty', False)}
    if not implicit:
        return dirty_nodes

    # Get implicitly dirty nodes (all of the packages that depend on a dirty package)
    dirty_nodes.update(*map(set, (graph.predecessors(n) for n in dirty_nodes)))
    return dirty_nodes

def build_order(graph, packages, level=0, filter_by_git_change=True):
    '''
    Assumes that packages are in graph.
    Builds a temporary graph of relevant nodes and returns it topological sort.

    Relevant nodes selected in a breadth first traversal sourced at each pkg in packages.

    Values expected for packages is one of None, sequence:
       None: build the whole graph
       empty sequence: build nodes marked dirty
       non-empty sequence: build nodes in sequence
    '''

    if packages is None and not filter_by_git_change:
        tmp_global = graph.subgraph(graph.nodes())
    else:
        if packages:
            packages = set(packages)
        else:
            packages = dirty(graph)
        tmp_global = graph.subgraph(packages)

        if level > 0:
            # for each level, add all deps
            _level = level

            currlevel = packages
            while _level > 0:
                newcurr = set()
                for p in currlevel:
                    newcurr.update(set(graph.successors(p)))
                    tmp_global.add_edges_from(graph.edges_iter(p))
                currlevel = newcurr
                _level -= 1

    #copy relevant node data to tmp_global
    for n in tmp_global.nodes_iter():
        tmp_global.node[n] = graph.node[n]

    return tmp_global, nx.topological_sort(tmp_global, reverse=True)

def make_deps(graph, package, dry=False, extra_args='', level=0, autofail=True):
    g, order = build_order(graph, package, level=level)
    # Filter out any packages that don't have recipes
    order = [pkg for pkg in order if g.node[pkg].get('meta')]
    print("Build order:\n{}".format('\n'.join(order)))

    failed = set()
    build_times = {x:None for x in order}
    for pkg in order:
        print("Building ", pkg)
        try:
            # Autofail package if any dependency build failed
            if any(p in failed for p in order):
                print(failed)
                failed_deps = [p for p in g.node[pkg]['meta']['depends'].keys() if p in failed]
                print("Building {} failed because one or more of its dependencies failed to build: ".format(pkg), end=' ')
                print(', '.join(failed_deps))
                failed.add(pkg)
                continue
            build_time = make_pkg(g.node[pkg], dry=dry, extra_args=extra_args)
            build_times[pkg] = build_time
            if build_times[pkg].returncode:
                failed.add(pkg)
        except KeyboardInterrupt:
            print('KeyboardInterrupt')
            break
        except subprocess.CalledProcessError:
            failed.add(pkg)
            continue

    return list(set(order)-failed), list(failed), build_times

def make_pkg(package, dry=False, extra_args=''):
    meta, path = package['meta'], package['recipe']
    print("===========> Building ", path)
    if not dry:
        try:
            extra_args = extra_args.split()
            args = ['conda', 'build', '-q'] + extra_args + [path]
            print("+ " + ' '.join(args))
            p = PopenWrapper(args, time_int=1)
            return p
        except subprocess.CalledProcessError as e:
            print("Build failed with errorcode: ", e.returncode)
            print(e)
            raise
    else:
        return PopenWrapper(['echo', '-dry', '(dry run)'])

def pre_build_clean_up(args):
    '''Copies files from patterns like:

    ./special_cases/<package-name>/run_test.sh

    to

    args.path/<package-name>/run_test.sh

    (Helpful if anaconda-build needs mods)
    '''
    special = os.path.join(os.path.dirname(__file__), 'special_cases')
    for dirr in os.listdir(special):
        for fil in os.listdir(os.path.join(special, dirr)):
            full_file = os.path.join(special, dirr, fil)
            if not os.path.exists(os.path.join(args.path, dirr)):
                continue
            target = os.path.join(args.path, dirr, fil)
            print('Copy', full_file, 'to', target)
            print('Copy', full_file, 'to', target+'_removed')
            shutil.copy(full_file, target + '_removed')
            shutil.copy(full_file, target)

def build_cli(parse_this=None):
    parser = argparse.ArgumentParser()
    parser.add_argument("path", default='.')
    build_pkgs = parser.add_mutually_exclusive_group()
    build_pkgs.add_argument("-build", action='append', default=[])
    build_pkgs.add_argument("-buildall", action='store_true')
    build_pkgs.add_argument('-json-file-key', default=[], nargs="+",
                            help="Example: -json-file-key package_tree.js libnetcdf pysam")
    build_pkgs.add_argument('--all-diffs',
                            help="Test any diffs in path including downstream",
                            action="store_true")
    parser.add_argument("-dry", action='store_true', default=False,
                              help="Dry run")
    parser.add_argument("-api", action='store_true', dest='recompile',
                              default=False)
    parser.add_argument("-args", action='store', dest='cbargs', default='')
    parser.add_argument("-l", type=int, action='store', dest='level', default=0)
    parser.add_argument("-noautofail", action='store_false', dest='autofail', default=True)
    parser.add_argument('--targetnum', '-t',
                        type=int,
                        help="Target number of packages in each subtree-build.")
    parser.add_argument('--packages', '-p',
                        default=[],
                        nargs="+",
                        help="Rather than determine tree, build the --packages in order")
    if parse_this is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(parse_this)
    if not args.build:
        args.build = None
    print('Running build2.py with args of', args)
    if getattr(args, 'json_file_key', None):
        assert len(args.json_file_key) == 2, 'Should be 2 args: json_filename key'
    return args
<commit_msg>better return codes<commit_after>#!/usr/bin/env python
from __future__ import print_function, division

import argparse
from collections import defaultdict
import datetime
import json
import psutil
import os
import shutil
import subprocess
import time
import networkx as nx
import sys

from conda_build.metadata import parse, MetaData

CONDA_BUILD_CACHE=os.environ.get("CONDA_BUILD_CACHE")

class PopenWrapper(object):
    # Small wrapper around subprocess.Popen to allow memory usage monitoring

    def __init__(self, *args, **kwargs):
        self.elapsed = None
        self.rss = None
        self.vms = None
        # set returncode to a bad one
        # in case it is never defined
        # after here.
        self.returncode = 173
        self.disk = None

        self._execute(*args, **kwargs)

    def _execute(self, *args, **kwargs):
        # The polling interval (in seconds)
        time_int = kwargs.pop('time_int', 1)

        # Create a process of this (the parent) process
        parent = psutil.Process(os.getpid())
        initial_usage = psutil.disk_usage(sys.prefix).used

        # Using the convenience Popen class provided by psutil
        start_time = time.time()
        _popen = psutil.Popen(*args, **kwargs)
        try:
            while _popen.is_running():
                #We need to get all of the children of our process since our process spawns other processes
                # Collect all of the child processes

                try:
                    # We use the parent process to get mem usage of all spawned processes
                    child_pids = [_.memory_info() for _ in parent.children(recursive=True) if _.is_running()]
                    # Sum the memory usage of all the children together (2D columnwise sum)
                    rss, vms = [sum(_) for _ in zip(*child_pids)]

                    self.rss = max(rss, self.rss)
                    self.vms = max(vms, self.vms)

                    # Get disk usage
                    used_disk = initial_usage - psutil.disk_usage(sys.prefix).used
                    self.disk = max(used_disk, self.disk)

                except psutil.AccessDenied as e:
                    if _popen.status() == psutil.STATUS_ZOMBIE:
                        _popen.wait()

                time.sleep(time_int)
                self.elapsed = time.time() - start_time
                self.returncode = _popen.returncode
                if _popen.returncode is not None:
                    _popen.kill()
                    break
        except KeyboardInterrupt:
            _popen.kill()
            raise

    def __repr__(self):
        return str({'elapsed': self.elapsed,
                    'rss': self.rss,
                    'vms': self.vms,
                    'returncode': self.returncode})

def bytes2human(n):
    # http://code.activestate.com/recipes/578019
    # >>> bytes2human(10000)
    # '9.8K'
    # >>> bytes2human(100001221)
    # '95.4M'
    symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
    prefix = {}
    for i, s in enumerate(symbols):
        prefix[s] = 1 << (i + 1) * 10
    for s in reversed(symbols):
        if n >= prefix[s]:
            value = float(n) / prefix[s]
            return '%.1f%s' % (value, s)
    return "%sB" % n

def last_changed_git_branch(git_root):
    args = ['git', 'for-each-ref',
            '--sort=-committerdate', 'refs/heads/',]
    proc = subprocess.Popen(args,
                            cwd=git_root,
                            stdout=subprocess.PIPE)
    if proc.wait():
        raise ValueError('Bad return code '
                         'from git branch sort', proc.poll())
    head_1 = proc.stdout.read().decode().splitlines()[0]
    branch = head_1.split()[-1]
    print('Last changed branch: ', branch)
    return branch

def git_changed_files(git_rev, git_root=''):
    """
    Get the list of files changed in a git revision and return a list of package directories that have been modified.
    """
    proc = subprocess.Popen(['git', 'diff-tree',
                              '--no-commit-id', '--name-only',
                              '-r', git_rev
                              ],
                              cwd=git_root,
                              stdout=subprocess.PIPE)
    if proc.wait():
        raise ValueError('Bad git return code: {}'.format(proc.poll()))
    files = proc.stdout.read().decode().splitlines()
    too_short = ('\\','/')
    changed = {os.path.dirname(f) for f in files}
    changed = {f for f in changed if f and f not in too_short}
    return changed

def read_recipe(path):
    return MetaData(path)

def describe_meta(meta):
    """Return a dictionary that describes build info of meta.yaml"""

    # Things we care about and need fast access to:
    #   1. Package name and version
    #   2. Build requirements
    #   3. Build number
    #   4. Recipe directory
    d = {}

    d['build'] = meta.get_value('build/number', 0)
    d['depends'] = format_deps(meta.get_value('requirements/build'))
    d['version'] = meta.get_value('package/version')
    return d

def format_deps(deps):
    d = {}
    for x in deps:
        x = x.strip().split()
        if len(x) == 2:
            d[x[0]] = x[1]
        else:
            d[x[0]] = ''
    return d

def get_build_deps(recipe):
    return format_deps(recipe.get_value('requirements/build'))

def construct_graph(directory, filter_by_git_change=True):
    '''
    Construct a directed graph of dependencies from a directory of recipes

    Annotate dependencies that don't have recipes in that directory
    '''
    print('construct_graph with args: ', directory, filter_by_git_change)
    g = nx.DiGraph()
    build_numbers = {}
    directory = os.path.abspath(directory)
    assert os.path.isdir(directory)

    # get all immediate subdirectories
    recipe_dirs = next(os.walk(directory))[1]
    recipe_dirs = set(x for x in recipe_dirs if not x.startswith('.'))
    if filter_by_git_change:
        changed_recipes = git_changed_files('HEAD', git_root=directory)
        print('changed_recipes {}'.format(changed_recipes))
    for rd in recipe_dirs:
        recipe_dir = os.path.join(directory, rd)
        try:
            pkg = read_recipe(recipe_dir)
            name = pkg.name()
        except:
            continue

        # add package (in case it has no build deps)
        if filter_by_git_change:
            _dirty = True if rd in changed_recipes else False
        else:
            _dirty = True
        g.add_node(name, meta=describe_meta(pkg), recipe=recipe_dir, dirty=_dirty)
        for k, d in get_build_deps(pkg).iteritems():
            g.add_edge(name, k)
    return g

def dirty(graph, implicit=True):
    """
    Return a set of all dirty nodes in the graph.

    These include implicit and explicit dirty nodes.
    """
    # Reverse the edges to get true dependency
    dirty_nodes = {n for n, v in graph.node.items() if v.get('dirty', False)}
    if not implicit:
        return dirty_nodes

    # Get implicitly dirty nodes (all of the packages that depend on a dirty package)
    dirty_nodes.update(*map(set, (graph.predecessors(n) for n in dirty_nodes)))
    return dirty_nodes

def build_order(graph, packages, level=0, filter_by_git_change=True):
    '''
    Assumes that packages are in graph.
    Builds a temporary graph of relevant nodes and returns it topological sort.

    Relevant nodes selected in a breadth first traversal sourced at each pkg in packages.

    Values expected for packages is one of None, sequence:
       None: build the whole graph
       empty sequence: build nodes marked dirty
       non-empty sequence: build nodes in sequence
    '''

    if packages is None and not filter_by_git_change:
        tmp_global = graph.subgraph(graph.nodes())
    else:
        if packages:
            packages = set(packages)
        else:
            packages = dirty(graph)
        tmp_global = graph.subgraph(packages)

        if level > 0:
            # for each level, add all deps
            _level = level

            currlevel = packages
            while _level > 0:
                newcurr = set()
                for p in currlevel:
                    newcurr.update(set(graph.successors(p)))
                    tmp_global.add_edges_from(graph.edges_iter(p))
                currlevel = newcurr
                _level -= 1

    #copy relevant node data to tmp_global
    for n in tmp_global.nodes_iter():
        tmp_global.node[n] = graph.node[n]

    return tmp_global, nx.topological_sort(tmp_global, reverse=True)

def make_deps(graph, package, dry=False, extra_args='', level=0, autofail=True):
    g, order = build_order(graph, package, level=level)
    # Filter out any packages that don't have recipes
    order = [pkg for pkg in order if g.node[pkg].get('meta')]
    print("Build order:\n{}".format('\n'.join(order)))

    failed = set()
    build_times = {x:None for x in order}
    for pkg in order:
        print("Building ", pkg)
        try:
            # Autofail package if any dependency build failed
            if any(p in failed for p in order):
                print(failed)
                failed_deps = [p for p in g.node[pkg]['meta']['depends'].keys() if p in failed]
                print("Building {} failed because one or more of its dependencies failed to build: ".format(pkg), end=' ')
                print(', '.join(failed_deps))
                failed.add(pkg)
                continue
            build_time = make_pkg(g.node[pkg], dry=dry, extra_args=extra_args)
            build_times[pkg] = build_time
            if build_times[pkg].returncode:
                failed.add(pkg)
        except KeyboardInterrupt:
            print('KeyboardInterrupt')
            break
        except subprocess.CalledProcessError:
            failed.add(pkg)
            continue

    return list(set(order)-failed), list(failed), build_times

def make_pkg(package, dry=False, extra_args=''):
    meta, path = package['meta'], package['recipe']
    print("===========> Building ", path)
    if not dry:
        try:
            extra_args = extra_args.split()
            args = ['conda', 'build', '-q'] + extra_args + [path]
            print("+ " + ' '.join(args))
            p = PopenWrapper(args, time_int=1)
            return p
        except subprocess.CalledProcessError as e:
            print("Build failed with errorcode: ", e.returncode)
            print(e)
            raise
    else:
        return PopenWrapper(['echo', '-dry', '(dry run)'])

def pre_build_clean_up(args):
    '''Copies files from patterns like:

    ./special_cases/<package-name>/run_test.sh

    to

    args.path/<package-name>/run_test.sh

    (Helpful if anaconda-build needs mods)
    '''
    special = os.path.join(os.path.dirname(__file__), 'special_cases')
    for dirr in os.listdir(special):
        for fil in os.listdir(os.path.join(special, dirr)):
            full_file = os.path.join(special, dirr, fil)
            if not os.path.exists(os.path.join(args.path, dirr)):
                continue
            target = os.path.join(args.path, dirr, fil)
            print('Copy', full_file, 'to', target)
            print('Copy', full_file, 'to', target+'_removed')
            shutil.copy(full_file, target + '_removed')
            shutil.copy(full_file, target)

def build_cli(parse_this=None):
    parser = argparse.ArgumentParser()
    parser.add_argument("path", default='.')
    build_pkgs = parser.add_mutually_exclusive_group()
    build_pkgs.add_argument("-build", action='append', default=[])
    build_pkgs.add_argument("-buildall", action='store_true')
    build_pkgs.add_argument('-json-file-key', default=[], nargs="+",
                            help="Example: -json-file-key package_tree.js libnetcdf pysam")
    build_pkgs.add_argument('--all-diffs',
                            help="Test any diffs in path including downstream",
                            action="store_true")
    parser.add_argument("-dry", action='store_true', default=False,
                              help="Dry run")
    parser.add_argument("-api", action='store_true', dest='recompile',
                              default=False)
    parser.add_argument("-args", action='store', dest='cbargs', default='')
    parser.add_argument("-l", type=int, action='store', dest='level', default=0)
    parser.add_argument("-noautofail", action='store_false', dest='autofail', default=True)
    parser.add_argument('--targetnum', '-t',
                        type=int,
                        help="Target number of packages in each subtree-build.")
    parser.add_argument('--packages', '-p',
                        default=[],
                        nargs="+",
                        help="Rather than determine tree, build the --packages in order")
    if parse_this is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(parse_this)
    if not args.build:
        args.build = None
    print('Running build2.py with args of', args)
    if getattr(args, 'json_file_key', None):
        assert len(args.json_file_key) == 2, 'Should be 2 args: json_filename key'
    return args
<|endoftext|>


<commit_before>```
Given a collection of numbers, return all possible permutations.

For example,
[1,2,3] have the following permutations:
[1,2,3], [1,3,2], [2,1,3], [2,3,1], [3,1,2], and [3,2,1].
```

class Solution:
    # @param {integer[]} nums
    # @return {integer[][]}
    # Recursion
    def permute_1(self, nums):
        result = []
        if len(nums) == 1: return [nums]
        else:
            for i in xrange(len(nums)):
                for ele in self.permute(nums[:i] + nums[i+1:]):
                    result.append([nums[i]] + ele)
            return result

    # Iteration
    def permute(self, num):
            result = [[]]
            for i in num:
                permutations = []  # Note: here permutations need to clean
                if result == [[]]:
                    result = [[i]]
                else :
                    for j in result :
                        for k in range(len(j)+1) :  # Note: this for loop should be after temp = j[:]
                            temp = j[:]
                            temp.insert(k,i)
                            permutations.append(temp)
                    result = permutations[:]
            return result
<commit_msg>Update PermutationsI.py<commit_after>```
Given a collection of numbers, return all possible permutations.

For example,
[1,2,3] have the following permutations:
[1,2,3], [1,3,2], [2,1,3], [2,3,1], [3,1,2], and [3,2,1].
```

class Solution:
    # @param {integer[]} nums
    # @return {integer[][]}
    # Recursion
    def permute_1(self, nums):
        result = []
        if len(nums) == 1: return [nums]
        else:
            for i in xrange(len(nums)):
                for ele in self.permute(nums[:i] + nums[i+1:]):
                    result.append([nums[i]] + ele)
            return result

    # Iteration
    def permute(self, num):
            result = [[]]
            for i in num:
                permutations = []  # Note: here permutations need to clean
                if result == [[]]:
                    result = [[i]]
                else :
                    for j in result :
                        for k in range(len(j)+1) :  # Note: this for loop should be after temp = j[:]
                            temp = j[:]
                            temp.insert(k,i)
                            permutations.append(temp)
                    result = permutations[:]
            return result

# Time complexity: O(n * n!) n!: n factorial
# Without going into the details of any specific algorithm, an optimal permutation algorithm has to generate all the possible n!
# permutation, each with n characters. So the best time complexity one can attain would be O(n∗n!), which is same as the complexity
# of almost all the permutation algorithms/code you can find online.
<|endoftext|>


<commit_before>"""
A simple command-line interface for Mininet.

The Mininet CLI provides a simple control console which
makes it easy to talk to nodes. For example, the command

mininet> h27 ifconfig

runs 'ifconfig' on host h27.

Having a single console rather than, for example, an xterm for each
node is particularly convenient for networks of any reasonable
size.

The CLI automatically substitutes IP addresses for node names,
so commands like

mininet> h0 ping -c1 h31

should work correctly and allow host h0 to ping host h31.
Note the '-c1' argument as per the Bugs/limitations section below!

Several useful commands are provided, including the ability to
list all nodes ('nodes'), to print out the network topology
('net') and to check connectivity ('pingall', 'pingpair')
and bandwidth ('iperf'.)

Bugs/limitations:

- Interactive commands are not supported at the moment;
  notably, if you type 'ping h1', you can control-C it, but
  it breaks the CLI and your network. ;-(
  For now, we recommend limiting CLI use to non-interactive
  commands which terminate in a reasonable amount of time.

"""

from subprocess import call
from cmd import Cmd

from mininet.log import info, output

class CLI( Cmd ):
    "Simple command-line interface to talk to nodes."

    prompt = 'mininet> '

    def __init__( self, mininet ):
        self.mn = mininet
        self.nodelist = self.mn.controllers + self.mn.switches + self.mn.hosts
        self.nodemap = {} # map names to Node objects
        for node in self.nodelist:
            self.nodemap[ node.name ] = node
        Cmd.__init__( self )
        info( '*** Starting CLI:\n' )
        self.cmdloop()

    # Disable pylint "Unused argument: 'arg's'" messages.
    # Each CLI function needs the same interface.
    # pylint: disable-msg=W0613

    def do_help( self, args ):
        "Describe available CLI commands."
        Cmd.do_help( self, args )
        helpStr = ( 'You may also send a command to a node using:\n'
                   '  <node> command {args}\n'
                   'For example:\n'
                   '  mininet> h0 ifconfig\n'
                   '\n'
                   'The interpreter automatically substitutes IP '
                   'addresses\n'
                   'for node names when a node is the first arg, so commands'
                   ' like\n'
                   ' mininet> h0 ping -c1 h1\n'
                   'should work.\n'
                   '\n'
                   'Interactive commands are not really supported yet,\n'
                   'so please limit commands to ones that do not\n'
                   'require user interaction and will terminate\n'
                   'after a reasonable amount of time.\n' )
        if args is "":
            self.stdout.write( helpStr )

    def do_nodes( self, args ):
        "List all nodes."
        nodes = ' '.join( [ node.name for node in sorted( self.nodelist ) ] )
        output( 'available nodes are: \n%s\n' % nodes )

    def do_net( self, args ):
        "List network connections."
        for switch in self.mn.switches:
            output( switch.name, '<->' )
            for intf in switch.intfs.values():
                name = switch.connection[ intf ][ 1 ]
                output( ' %s' % name )
            output( '\n' )

    def do_sh( self, args ):
        "Run an external shell command"
        call( args, shell=True )

    # do_py() needs to catch any exception during eval()
    # pylint: disable-msg=W0703

    def do_py( self, args ):
        """Evaluate a Python expression.
           Node names may be used, e.g.: h1.cmd('ls')"""
        try:
            result = eval( args, globals(), self.nodemap )
            if not result:
                return
            elif isinstance( result, str ):
                info( result + '\n' )
            else:
                info( repr( result ) + '\n' )
        except Exception, e:
            info( str( e ) + '\n' )

    # pylint: enable-msg=W0703

    def do_pingall( self, args ):
        "Ping between all hosts."
        self.mn.pingAll()

    def do_pingpair( self, args ):
        "Ping between first two hosts, useful for testing."
        self.mn.pingPair()

    def do_iperf( self, args ):
        "Simple iperf TCP test between two hosts."
        self.mn.iperf()

    def do_iperfudp( self, args ):
        "Simple iperf UDP test between two hosts."
        udpBw = args[ 0 ] if len( args ) else '10M'
        self.mn.iperfUdp( udpBw )

    def do_intfs( self, args ):
        "List interfaces."
        for node in self.nodelist:
            output( '%s: %s\n' %
                ( node.name, ' '.join( sorted( node.intfs.values() ) ) ) )

    def do_dump( self, args ):
        "Dump node info."
        for node in self.nodelist:
            output( '%s\n' % node )

    def do_exit( self, args ):
        "Exit"
        return 'exited by user command'

    def do_quit( self, args ):
        "Exit"
        return self.do_exit( args )

    def do_EOF( self, args ):
        "Exit"
        return self.do_exit( args )

    def default( self, line ):
        """Called on an input line when the command prefix is not recognized.
        Overridden to run shell commands when a node is the first CLI argument.
        Past the first CLI argument, node names are automatically replaced with
        corresponding IP addrs."""

        first, args, line = self.parseline( line )
        if len(args) > 0 and args[ -1 ] == '\n':
            args = args[ :-1 ]
        rest = args.split( ' ' )

        if first in self.nodemap:
            node = self.nodemap[ first ]
            # Substitute IP addresses for node names in command
            rest = [ self.nodemap[ arg ].IP()
                    if arg in self.nodemap else arg
                    for arg in rest ]
            rest = ' '.join( rest )
            # Run cmd on node:
            node.sendCmd( rest )
            while True:
                try:
                    done, data = node.monitor()
                    info( '%s' % data )
                    if done:
                        break
                except KeyboardInterrupt:
                    node.sendInt()
        else:
            self.stdout.write( '*** Unknown syntax: %s\n' % line )

    # Re-enable pylint "Unused argument: 'arg's'" messages.
    # pylint: enable-msg=W0613
<commit_msg>Changes to note neither ^C nor interactive commands work atm.<commit_after>"""
A simple command-line interface for Mininet.

The Mininet CLI provides a simple control console which
makes it easy to talk to nodes. For example, the command

mininet> h27 ifconfig

runs 'ifconfig' on host h27.

Having a single console rather than, for example, an xterm for each
node is particularly convenient for networks of any reasonable
size.

The CLI automatically substitutes IP addresses for node names,
so commands like

mininet> h0 ping -c1 h31

should work correctly and allow host h0 to ping host h31.
Note the '-c1' argument as per the Bugs/limitations section below!

Several useful commands are provided, including the ability to
list all nodes ('nodes'), to print out the network topology
('net') and to check connectivity ('pingall', 'pingpair')
and bandwidth ('iperf'.)

Bugs/limitations:

- Interactive commands are not supported at the moment;
  notably, if you type 'ping h1', you can't interrupt it.
  For now, we recommend limiting CLI use to non-interactive
  commands which terminate in a reasonable amount of time.

"""

from subprocess import call
from cmd import Cmd

from mininet.log import info, output

class CLI( Cmd ):
    "Simple command-line interface to talk to nodes."

    prompt = 'mininet> '

    def __init__( self, mininet ):
        self.mn = mininet
        self.nodelist = self.mn.controllers + self.mn.switches + self.mn.hosts
        self.nodemap = {} # map names to Node objects
        for node in self.nodelist:
            self.nodemap[ node.name ] = node
        Cmd.__init__( self )
        info( '*** Starting CLI:\n' )
        self.cmdloop()

    # Disable pylint "Unused argument: 'arg's'" messages.
    # Each CLI function needs the same interface.
    # pylint: disable-msg=W0613

    def do_help( self, args ):
        "Describe available CLI commands."
        Cmd.do_help( self, args )
        helpStr = ( 'You may also send a command to a node using:\n'
                   '  <node> command {args}\n'
                   'For example:\n'
                   '  mininet> h0 ifconfig\n'
                   '\n'
                   'The interpreter automatically substitutes IP '
                   'addresses\n'
                   'for node names when a node is the first arg, so commands'
                   ' like\n'
                   ' mininet> h0 ping -c1 h1\n'
                   'should work.\n'
                   '\n'
                   'Interactive commands are not supported yet,\n'
                   'so please limit commands to ones that do not\n'
                   'require user interaction and will terminate\n'
                   'after a reasonable amount of time.\n' )
        if args is "":
            self.stdout.write( helpStr )

    def do_nodes( self, args ):
        "List all nodes."
        nodes = ' '.join( [ node.name for node in sorted( self.nodelist ) ] )
        output( 'available nodes are: \n%s\n' % nodes )

    def do_net( self, args ):
        "List network connections."
        for switch in self.mn.switches:
            output( switch.name, '<->' )
            for intf in switch.intfs.values():
                name = switch.connection[ intf ][ 1 ]
                output( ' %s' % name )
            output( '\n' )

    def do_sh( self, args ):
        "Run an external shell command"
        call( args, shell=True )

    # do_py() needs to catch any exception during eval()
    # pylint: disable-msg=W0703

    def do_py( self, args ):
        """Evaluate a Python expression.
           Node names may be used, e.g.: h1.cmd('ls')"""
        try:
            result = eval( args, globals(), self.nodemap )
            if not result:
                return
            elif isinstance( result, str ):
                info( result + '\n' )
            else:
                info( repr( result ) + '\n' )
        except Exception, e:
            info( str( e ) + '\n' )

    # pylint: enable-msg=W0703

    def do_pingall( self, args ):
        "Ping between all hosts."
        self.mn.pingAll()

    def do_pingpair( self, args ):
        "Ping between first two hosts, useful for testing."
        self.mn.pingPair()

    def do_iperf( self, args ):
        "Simple iperf TCP test between two hosts."
        self.mn.iperf()

    def do_iperfudp( self, args ):
        "Simple iperf UDP test between two hosts."
        udpBw = args[ 0 ] if len( args ) else '10M'
        self.mn.iperfUdp( udpBw )

    def do_intfs( self, args ):
        "List interfaces."
        for node in self.nodelist:
            output( '%s: %s\n' %
                ( node.name, ' '.join( sorted( node.intfs.values() ) ) ) )

    def do_dump( self, args ):
        "Dump node info."
        for node in self.nodelist:
            output( '%s\n' % node )

    def do_exit( self, args ):
        "Exit"
        return 'exited by user command'

    def do_quit( self, args ):
        "Exit"
        return self.do_exit( args )

    def do_EOF( self, args ):
        "Exit"
        return self.do_exit( args )

    def default( self, line ):
        """Called on an input line when the command prefix is not recognized.
        Overridden to run shell commands when a node is the first CLI argument.
        Past the first CLI argument, node names are automatically replaced with
        corresponding IP addrs."""

        first, args, line = self.parseline( line )
        if len(args) > 0 and args[ -1 ] == '\n':
            args = args[ :-1 ]
        rest = args.split( ' ' )

        if first in self.nodemap:
            node = self.nodemap[ first ]
            # Substitute IP addresses for node names in command
            rest = [ self.nodemap[ arg ].IP()
                    if arg in self.nodemap else arg
                    for arg in rest ]
            rest = ' '.join( rest )
            # Run cmd on node:
            node.sendCmd( rest )
            while True:
                try:
                    done, data = node.monitor()
                    info( '%s' % data )
                    if done:
                        break
                except KeyboardInterrupt:
                    node.sendInt()
        else:
            self.stdout.write( '*** Unknown syntax: %s\n' % line )

    # Re-enable pylint "Unused argument: 'arg's'" messages.
    # pylint: enable-msg=W0613
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2018 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

from collections import defaultdict
import math
import sys
import threading

import kevlar
import khmer
import numpy

class KevlarZeroAbundanceDistError(ValueError):
    pass

def weighted_mean_std_dev(values, weights):
    mu = numpy.average(values, weights=weights)
    sigma = math.sqrt(numpy.average((values-mu)**2, weights=weights))
    return mu, sigma

def dist(infiles, mask, ksize=31, memory=1e6, threads=1, logstream=sys.stderr):
    message = 'Allocating memory for k-mer counts...'
    print('[kevlar::dist]', message, end='', file=logstream)
    counts = khmer.Counttable(ksize, memory / 4, 4)
    tracking = khmer.Nodetable(ksize, 1, 1, primes=counttable.hashsizes())
    print('done!', file=logstream)

    message = 'Processing input with {:d} threads'.format(threads)
    print('[kevlar::dist]', message, file=logstream)
    for filename in infiles:
        print('    -', filename, file=logstream)
        parser = khmer.ReadParser(filename)
        threads = list()
        for _ in range(threads):
            thread = threading.Thread(
                target=counts.consume_seqfile_with_mask,
                args=(parser, mask,),
                kwargs={'threshold': 1, 'complement': True},
            )
            threads.append(thread)
            thread.start()
        for thread in threads:
            thread.join()
    print('[kevlar::dist] Done processing input!', file=logstream)

    print('[kevlar::dist] Second pass over the data', file=logstream)
    abund_lists = list()

    def __do_abund_dist(parser):
        abund = counts.abundance_distribution(parser, tracking)
        abund_lists.append(abund)
    for filename in infiles:
        print('    -', filename, file=logstream)
        parser = khmer.ReadParser(filename)
        threads = list()
        for _ in range(threads):
            thread = threading.Thread(
                target=__do_abund_dist,
                args=(parser,)
            )
            threads.append(thread)
            thread.start()
        for thread in threads:
            thread.join()

    assert len(abund_lists) == len(infiles) * threads
    abundance = defaultdict(int)
    for abund in abund_lists:
        for i, count in enumerate(abund):
            abundance[i] += count

    message = 'ignoring {:d} k-mers as not in mask'.format(abundance[0])
    print('[kevlar::dist]', message, file=logstream)
    del(abundance[0])
    total = sum(abundance.values())
    if total == 0:
        message = 'all k-mer abundances are 0, please check input files'
        raise KevlarZeroAbundanceDistError(message)
    mu, sigma = weighted_mean_std_dev(abundance.keys(), abundance.values())
    return mu, sigma
<commit_msg>Quick refactor, add dist calcuation<commit_after>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2018 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

from collections import defaultdict
import math
import sys
import threading

import kevlar
import khmer
import numpy

class KevlarZeroAbundanceDistError(ValueError):
    pass

def weighted_mean_std_dev(values, weights):
    mu = numpy.average(values, weights=weights)
    sigma = math.sqrt(numpy.average((values-mu)**2, weights=weights))
    return mu, sigma

def count_first_pass(infiles, counts, mask, threads=1, logstream=sys.stderr):
    message = 'Processing input with {:d} threads'.format(threads)
    print('[kevlar::dist]', message, file=logstream)

    for filename in infiles:
        print('    -', filename, file=logstream)
        parser = khmer.ReadParser(filename)
        threads = list()
        for _ in range(threads):
            thread = threading.Thread(
                target=counts.consume_seqfile_with_mask,
                args=(parser, mask,),
                kwargs={'threshold': 1, 'complement': True},
            )
            threads.append(thread)
            thread.start()
        for thread in threads:
            thread.join()

    print('[kevlar::dist] Done processing input!', file=logstream)

def count_second_pass(infiles, counts, threads=1, logstream=logstream):
    print('[kevlar::dist] Second pass over the data', file=logstream)
    tracking = khmer.Nodetable(ksize, 1, 1, primes=counttable.hashsizes())
    abund_lists = list()

    def __do_abund_dist(parser):
        abund = counts.abundance_distribution(parser, tracking)
        abund_lists.append(abund)

    for filename in infiles:
        print('    -', filename, file=logstream)
        parser = khmer.ReadParser(filename)
        threads = list()
        for _ in range(threads):
            thread = threading.Thread(
                target=__do_abund_dist,
                args=(parser,)
            )
            threads.append(thread)
            thread.start()
        for thread in threads:
            thread.join()

    assert len(abund_lists) == len(infiles) * threads
    abundance = defaultdict(int)
    for abund in abund_lists:
        for i, count in enumerate(abund):
            abundance[i] += count

    print('[kevlar::dist] Done second pass over input!', file=logstream)

    return abundance

def calc_mu_sigma(abundance):
    message = 'ignoring {:d} k-mers as not in mask'.format(abundance[0])
    print('[kevlar::dist]', message, file=logstream)
    del(abundance[0])
    total = sum(abundance.values())
    if total == 0:
        message = 'all k-mer abundances are 0, please check input files'
        raise KevlarZeroAbundanceDistError(message)
    mu, sigma = weighted_mean_std_dev(abundance.keys(), abundance.values())
    return mu_sigma

def compute_dist(abundance):
    total = sum(abundance.values())
    fields = ['Abundance', 'Count', 'CumulativeCount', 'CumulativeFraction']
    data = pandas.DataFrame(columns=fields)
    cuml = 0
    for abund, count in sorted(abundance.items()):
        if count == 0:
            continue
        cuml += count
        frac = cuml / total
        row = {
            'Abundance': abund,
            'Count': count,
            'CumulativeCount': cuml,
            'CumulativeFraction': frac,
        }
        data = data.append(row, ignore_index=True)
    return abund

def dist(infiles, mask, ksize=31, memory=1e6, threads=1, logstream=sys.stderr):
    counts = khmer.Counttable(ksize, memory / 4, 4)
    count_first_pass(infiles, counts, mask, threads=threads,
                     logstream=logstream)
    abundance = count_second_pass(infiles, counts, threads=threads,
                                  logstream=logstream)
    mu, sigma = calc_mu_sigma(abundance)
    data = compute_dist(abundance)
    return mu, sigma, data
<|endoftext|>


<commit_before>#!/usr/bin/python

# keyanalyzer.py
#   Author: Austin Voecks

# example usage:
#   python keyanalyzer.py ~/.keyview.log 50 1-5

import sys, operator

def get_substrings(lines, range_start, range_end):
    ''' list of strings, int, int -> dict of strings to lists of ints

    fills a dictionary with the frequencies of substrings, where the substrings
    are sizes from @range_start to @range_end. this is completed in one pass
    '''
    substrings = {}

    for line_index in xrange(len(lines)):
        base = '\t'.join(lines[line_index:line_index + range_start])

        for offset in xrange(range_start, range_end):
            try:
                substrings[base][0] += 1
            except KeyError:
                substrings[base] = [1, offset]

            try:
                base += '\t' + lines[line_index + offset]
            except IndexError:
                pass

    return substrings

def get_total(substrings):
    ''' dict of strings to lists of ints -> float
    '''
    total = 0.0

    for key in substrings:
        if substrings[key][1] == 1:
            total += substrings[key][0]

    return total

def print_substrings(sorted_substrings, num_results, total_keys):
    ''' dict of strings to lists of ints, float -> IO
    '''
    for substring in sorted_substrings[-num_results:][::-1]:

        key_sequence   = substring[0].replace('\t', ' ')
        sequence_count = substring[1][0]

        if total_keys:
            ratio = (substring[1][0] / total_keys) * 100
            print("%-40s%-15d%6.2f" % (key_sequence, sequence_count, ratio))

        else:
            print("%-40s%-15d" % (key_sequence, sequence_count))

    print('\ntotal_keys keys: ' + str(int(total_keys)))

def main():
    ''' string, int, [int | int-int] -> none
    '''
    # agruments
    try:
        file_name      = sys.argv[1]
        num_results    = int(sys.argv[2])
        substring_size = sys.argv[3]

        # given a range or single value?
        try:
            start, end = [int(x) for x in substring_size.split('-')]
        except ValueError:
            start = end = int(substring_size)
        end += 1

    except IndexError:
        print('usage: keyboard_device top_n range')
        sys.exit(1)

    # get file contents
    with open(file_name) as f:
        lines = [x.rstrip(' \n') for x in f.readlines()]

    # get substrings, total keys
    substrings = get_substrings(lines, start, end)
    total_keys = get_total(substrings)

    # sort substrings by frequency
    sorted_substrings = sorted(substrings.items(), key=operator.itemgetter(1))

    # final output
    print_substrings(sorted_substrings, num_results, total_keys)

if __name__ == '__main__':
    main()
<commit_msg>Keyanalyzer now uses a query REPL!<commit_after>#!/usr/bin/python

# keyanalyzer.py
#   Author: Austin Voecks

# example usage:
#   python keyanalyzer.py ~/.keyview.log 50 1-5

import sys, operator, time

help_text = \
    'h         : help'\
    'r int-int : set substring size range'\
    's int     : set number of results'\
    'd         : default query'\
    'm str ... : query, results must match provided input'\
    'f str ... : query, results must not match provided input'\
    't str     : query, results must match provided input exactly'

def get_substrings(lines, range_start, range_end):
    ''' list of strings, int, int -> dict of strings to lists of ints

    fills a dictionary with the frequencies of substrings, where the substrings
    are sizes from @range_start to @range_end. this is completed in one pass
    '''
    substrings = {}
    offsets = xrange(range_start, range_end)

    for line_index in xrange(len(lines)):
        base = '\t'.join(lines[line_index:line_index + range_start])

        for offset in offsets:
            try:
                substrings[base][0] += 1
            except KeyError:
                substrings[base] = [1, offset]

            try:
                base += '\t' + lines[line_index + offset]
            except IndexError:
                pass

    return substrings

def get_total(substrings):
    ''' dict of strings to lists of ints -> float
    '''
    total = 0.0

    for key in substrings:
        if substrings[key][1] == 1:
            total += substrings[key][0]

    return total

def shorthand(substring):
    ''' string -> string
    '''
    out  = ''
    keys = substring.split('\t')
    printed = []

    for key in keys:
        if key not in printed:
            cnt = keys.count(key)
            printed.append(key)

            if cnt > 1:
                out += key + ' x' + str(cnt) + ' '
            else:
                out += key + ' '

    return out

def print_substrings(sorted_substrings, num_results, total_keys):
    ''' dict of strings to lists of ints, float -> IO
    '''
    for substring in sorted_substrings[-num_results:][::-1]:

        key_sequence   = shorthand(substring[0])
        sequence_count = substring[1][0]

        if total_keys:
            ratio = (substring[1][0] / total_keys) * 100
            print("%-50s%-15d%6.2f" % (key_sequence[:50], sequence_count, ratio))

        else:
            print("%-50s%-15d" % (key_sequence[:50], sequence_count))

def filter_dict(condition, dictionary):
    ''' function, dict -> dict
    '''
    out = []
    for key in dictionary:
        if condition(key):
            out.append((key, dictionary[key]))
    return dict(out)

def main():
    ''' string -> IO
    '''
    # agruments
    try:
        file_name = sys.argv[1]
    except IndexError:
        print('usage: keyboard_device top_n range')
        return

    num_results = 25

    # get file contents
    with open(file_name) as f:
        lines = [x.rstrip(' \n') for x in f.readlines()]

    # get substrings, total keys
    substrings = get_substrings(lines, 1, 2)
    total_keys = get_total(substrings)

    print('keyanalyzer.py')
    print('\ttotal_keys : ' + str(int(total_keys)))
    print('\ttype \'h\' for help')

    # query engine
    while True:
        try:
            print
            query = raw_input('?> ')
            query_start = time.time() * 1000
            run_query = False

            # help
            if query[0] == 'h':
                print(help_text)

            # set: query range
            elif query[0] == 'r':
                try:
                    start, end = [int(x) for x in query[2:].split('-')]
                except ValueError:
                    start = end = int(query[2:])
                end += 1

                substrings = get_substrings(lines, start, end)

            # set: query size
            elif query[0] == 's':
                num_results = int(query.split(' ')[1])

            # query: default
            elif query[0] == 'd':
                condition = False
                run_query = True

            # query: match strings
            elif query[0] == 'm':
                matches = query[2:].split()
                condition = lambda key : all(x in key for x in matches)
                run_query = True

            # query: filter strings
            elif query[0] == 'f':
                filters = query[2:].split()
                condition = lambda key : all(x not in key for x in filters)
                run_query = True

            # query: find string
            elif query[0] == 't':
                string = query[2:]
                condition = lambda key : ''.join(key.split('\t')) == string
                run_query = True

            # run the requested query
            if run_query:
                if condition:
                    query_dict = filter_dict(condition, substrings)
                else:
                    query_dict = substrings

                results = sorted(query_dict.items(), key=operator.itemgetter(1))
                print_substrings(results, num_results, total_keys)

            print(' ' * 72 + '%3.3f ms' % (time.time() * 1000 - query_start))

        except (EOFError, KeyboardInterrupt):
            return

        except (ValueError):
            print('\ttype \'h\' for help')

if __name__ == '__main__':
    main()
<|endoftext|>