<commit_before>from skypyblue.models import Variable, Method, Constraint, Strength, InternalStrength
from skypyblue.mvine import Mvine
from skypyblue.marker import Marker
import pdb;

class ConstraintSystem:

  def __init__(self):
    self.marker = Marker()
    self.constraints = []
    self.variables = []
    self.forced_constraint = None
    self.mark = None

  def create_variables(self, names, initialValues):
    assert len(names) == len(initialValues)
    res = []
    for i in range(len(names)):
      res.append(self.create_variable(names[i], initialValues[i]))
    return res

  def create_variable(self, name, initialValue):
    variable = Variable(name, initialValue, self)
    self.variables.append(variable)
    return variable

  def change_variable_values(self, variables, values):
    if len(variables) == 1:
      values = values[0]

    m = Method([], variables,
      lambda: values)

    cn = Constraint(
      lambda x: True,
      InternalStrength.FORCED,
      variables,
      [m])

    if self.forced_constraint is not None:
      self.remove_constraint(self.forced_constraint)

    self.forced_constraint = cn
    self.add_constraint(cn)

  def variable_changed(self, var):
    self.change_variable_values([var], [var.value])

  def add_stay_constraint(self, variable, strength):
    m = Method([], [variable], lambda: variable.value)
    stay_constraint = Constraint(
      lambda x: True,
      strength,
      [variable],
      [m])

    self.add_constraint(stay_constraint)
    return stay_constraint;

  def add_constraint(self, constraint):
    constraint.selected_method = None
    constraint.mark = None

    for variable in constraint.variables:
      variable.add_constraint(constraint)
    exec_roots = self.update_method_graph([constraint])
    self.exec_from_roots(exec_roots)

  def remove_constraint(self, constraint):
    for variable in constraint.variables:
      variable.remove_constraint(constraint)

    if constraint.is_enforced():
      old_outputs = constraint.selected_method.outputs
      constraint.selected_method = None

      exec_roots = []
      for variable in old_outputs:
        variable.determined_by = None
        variable.walk_strength = Strength.WEAKEST
        exec_roots.append(variable)

      self.propagate_walk_strength(old_outputs)
      unenforcedConstraints = []
      self.collect_unenforced(unenforcedConstraints, old_outputs,constraint.strength,True)
      exec_roots = self.update_method_graph(unenforcedConstraints)
      self.exec_from_roots(exec_roots)

  def update_method_graph(self, unenforced_constraints):
    exec_roots = []
    while unenforced_constraints:
      cn = self.strongest_constraint(unenforced_constraints)
      unenforced_constraints = list(
        filter(lambda c: c != cn, unenforced_constraints))
      redetermined_vars = []
      ok = Mvine(self.marker).build(cn, redetermined_vars)
      if not ok:
        return exec_roots
      self.propagate_walk_strength([cn] + redetermined_vars)
      self.collect_unenforced(unenforced_constraints, redetermined_vars, cn.strength, False)
      exec_roots.append(cn)
      for var in redetermined_vars:
        if var.determined_by is None:
          exec_roots.append(var)
    return exec_roots

  def strongest_constraint(self, constraints):
    constraints.sort(key = lambda cn: cn.strength, reverse = True)
    return constraints[0]

  def weakest_constraint(self, constraints):
    constraints.sort(key=lambda cn: cn.strength)
    return constraints[0]

  def propagate_walk_strength(self, roots):
    prop_mark = self.marker.new_mark()
    walk_pplan = self.pplan_add([], roots, prop_mark)

    while walk_pplan:
      cn = walk_pplan.pop()
      if self.any_immediate_upstream_marked(cn, prop_mark):
        for var  in cn.selected_method.inputs:
          if var.determined_by != None:
            if var.determined_by.mark == prop_mark:
              var.walk_strength = Strength.WEAKEST
      self.compute_walkabout_strengths(cn)
      cn.mark = None

  def any_immediate_upstream_marked(self, cn, mark):
    for var in cn.selected_method.inputs:
        if var.determined_by != None and var.determined_by.mark == mark:
          return True
    return False

  def compute_walkabout_strengths(self, cn):
    current_outputs = cn.selected_method.outputs
    for out_var in current_outputs:
      min_strength = cn.strength
      for mt in cn.methods:
        if not out_var in mt.outputs:
          max_strength = self.max_out(mt, current_outputs)
          if Strength.weaker(max_strength, min_strength):
            min_strength = max_strength
      out_var.walk_strength = min_strength

  def collect_unenforced(self, unenforced_cns, vars, collection_strength, collect_equal_strength):
    done_mark = self.marker.new_mark()
    for var in vars:
      unenforced_cns.extend(self.collect_unenforced_mark(unenforced_cns, var, collection_strength, collect_equal_strength, done_mark))
    return unenforced_cns

  def collect_unenforced_mark(self, unenforced_cns, var, collection_strength, collect_equal_strength, done_mark):
    for cn in var.constraints:
      if cn != var.determined_by and cn.mark != done_mark:
        cn.mark = done_mark
        if cn.is_enforced():
          for out_var in cn.selected_method.outputs:
            self.collect_unenforced_mark(unenforced_cns, out_var, collection_strength, collect_equal_strength, done_mark)
        elif Strength.weaker(cn.strength, collection_strength) or \
              (collect_equal_strength and (cn.strength == collection_strength)):
          unenforced_cns.append(cn)
    return unenforced_cns

  def exec_pplan_create(self, exec_roots, mark):
    exec_pplan = []

    for cn in exec_roots:
      if isinstance(cn, Constraint):
        cn.add_to_pplan(exec_pplan, mark)

    for var in exec_roots:
      if isinstance(var, Variable):
        if var.determined_by == None and not var.valid:
          exec_pplan.extend(var.add_to_pplan([], mark))
          var.valid = True

    return exec_pplan

  def exec_from_roots(self, exec_roots):
    prop_mark = self.marker.new_mark()
    exec_pplan = self.exec_pplan_create(exec_roots, prop_mark)

    while exec_pplan:
      cn = exec_pplan.pop()
      if cn.mark != prop_mark:
        # this cn has already been processed: do nothing
        # WTF?
        continue
      elif self.any_immediate_upstream_marked(cn, prop_mark):
        self.exec_from_cycle(cn, prop_mark)
      else:
        cn.mark = None
        self.execute_propagate_valid(cn)

  def execute_propagate_valid(self, cn):
    inputs_valid = True
    for var in cn.selected_method.inputs:
      if not var.valid:
        inputs_valid = False
    if inputs_valid:
      cn.selected_method.execute()
    for var in cn.selected_method.outputs:
      var.valid = inputs_valid

  def exec_from_cycle(self, cn, prop_mark):
    if cn.mark == prop_mark:
      cn.mark = None
      for var in cn.selected_method.outputs:
        var.valid = False
        for consuming_cn in var.constraints:
          if consuming_cn != cn and consuming_cn.is_enforced:
            self.exec_from_cycle(consuming_cn, prop_mark)

  def pplan_add(self, pplan, objs, done_mark):
    if not isinstance(objs, list): raise Exception("accepting only list of objs!")
    for elt in objs:
      elt.add_to_pplan(pplan, done_mark)
    return pplan

  def _new_mark(self):
    self.marker.new_mark()

  @property
  def mark(self):
    return self.marker.mark

  # def extract_plan(self, root_cns):
  #   good_cns = []
  #   bad_cns = []
  #   prop_mark = self.marker.new_mark()
  #   pplan = self.pplan_add([], root_cns, prop_mark)

  #   while pplan:
  #     cn = pplan.pop()
  #     if cn.mark != prop_mark:
  #       pass
  #     elif self.any_immediate_upstream_marked(cn, prop_mark):
  #       bad_cns.append(cn)
  #     else:
  #       cn.mark = None
  #       good_cns.append(cn)
  #   return self.create_valid_plan(root_cns, good_cns, bad_cns)

  # def create_valid_plan(self, root_cns, good_cns, bad_cns):
  #   plan = Plan(root_cns, good_cns, bad_cns)
  #   for cns in [root_cns, good_cns, bad_cns]:
  #     for cn in cns:
  #       cn.valid_plans.append(plan)
  #   return plan

  # def invalidate_plans_on_setting_method(self, cn, new_mt):
  #   self.invalidate_constraint_plans(cl.valid_plans)
  #   if new_mt != None:
  #     for var in new_mt.inputs:
  #       if var.determined_by != None:
  #         self.invalidate_constraint_plans(var.determined_by)

  # def invalidate_constraint_plans(self, invalid_cn):
  #   for plan in invalid_cn.valid_plans:
  #     plan.valid = False
  #     for cns in [plan.good_cns, plan.bad_cns, plan.root_cns]:
  #       for cn in cns:
  #         if cn != invalid_cn:
  #           cn.valid_plans.remove(plan)
  #   invalid_cn.valid_plans.clear()

  # def execute_plan(self, plan):
  #   if plan.valid:
  #     for cn in plan.good_cns:
  #       self.execute_propagate_valid(cn)
  #   else:
  #     raise ValueError("trying to execute invalid plan")

  def max_out(self, mt, current_outputs):
    max_strength = Strength.WEAKEST
    for var in mt.outputs:
      if not var in current_outputs and \
        Strength.weaker(max_strength, var.walk_strength):
        max_strength = var.walk_strength
    return max_strength

<commit_msg>constraint_system: removed of marks through the methods<commit_after>from skypyblue.models import Variable, Method, Constraint, Strength, InternalStrength
from skypyblue.mvine import Mvine
from skypyblue.marker import Marker
import pdb;

class ConstraintSystem:

  def __init__(self):
    self.marker = Marker()
    self.constraints = []
    self.variables = []
    self.forced_constraint = None

  def create_variables(self, names, initialValues):
    assert len(names) == len(initialValues)
    res = []
    for i in range(len(names)):
      res.append(self.create_variable(names[i], initialValues[i]))
    return res

  def create_variable(self, name, initialValue):
    variable = Variable(name, initialValue, self)
    self.variables.append(variable)
    return variable

  def change_variable_values(self, variables, values):
    if len(variables) == 1:
      values = values[0]

    m = Method([], variables,
      lambda: values)

    cn = Constraint(
      lambda x: True,
      InternalStrength.FORCED,
      variables,
      [m])

    if self.forced_constraint is not None:
      self.remove_constraint(self.forced_constraint)

    self.forced_constraint = cn
    self.add_constraint(cn)

  def variable_changed(self, var):
    self.change_variable_values([var], [var.value])

  def add_stay_constraint(self, variable, strength):
    m = Method([], [variable], lambda: variable.value)
    stay_constraint = Constraint(
      lambda x: True,
      strength,
      [variable],
      [m])

    self.add_constraint(stay_constraint)
    return stay_constraint;

  def add_constraint(self, constraint):
    constraint.selected_method = None
    constraint.mark = None

    for variable in constraint.variables:
      variable.add_constraint(constraint)
    exec_roots = self.update_method_graph([constraint])
    self.exec_from_roots(exec_roots)

  def remove_constraint(self, constraint):
    for variable in constraint.variables:
      variable.remove_constraint(constraint)

    if constraint.is_enforced():
      old_outputs = constraint.selected_method.outputs
      constraint.selected_method = None

      exec_roots = []
      for variable in old_outputs:
        variable.determined_by = None
        variable.walk_strength = Strength.WEAKEST
        exec_roots.append(variable)

      self.propagate_walk_strength(old_outputs)
      unenforcedConstraints = []
      self.collect_unenforced(unenforcedConstraints, old_outputs,constraint.strength,True)
      exec_roots = self.update_method_graph(unenforcedConstraints)
      self.exec_from_roots(exec_roots)

  def update_method_graph(self, unenforced_constraints):
    exec_roots = []
    while unenforced_constraints:
      cn = self.strongest_constraint(unenforced_constraints)
      unenforced_constraints = list(
        filter(lambda c: c != cn, unenforced_constraints))
      redetermined_vars = []
      ok = Mvine(self.marker).build(cn, redetermined_vars)
      if not ok:
        return exec_roots
      self.propagate_walk_strength([cn] + redetermined_vars)
      self.collect_unenforced(unenforced_constraints, redetermined_vars, cn.strength, False)
      exec_roots.append(cn)
      for var in redetermined_vars:
        if var.determined_by is None:
          exec_roots.append(var)
    return exec_roots

  def strongest_constraint(self, constraints):
    constraints.sort(key = lambda cn: cn.strength, reverse = True)
    return constraints[0]

  def weakest_constraint(self, constraints):
    constraints.sort(key=lambda cn: cn.strength)
    return constraints[0]

  def propagate_walk_strength(self, roots):
    self._new_mark()
    walk_pplan = self.pplan_add([], roots)

    while walk_pplan:
      cn = walk_pplan.pop()
      if self.any_immediate_upstream_marked(cn):
        for var  in cn.selected_method.inputs:
          if var.determined_by != None:
            if var.determined_by.mark == self.mark:
              var.walk_strength = Strength.WEAKEST
      self.compute_walkabout_strengths(cn)
      cn.mark = None

  def any_immediate_upstream_marked(self, cn):
    for var in cn.selected_method.inputs:
        if var.determined_by != None and var.determined_by.mark == self.mark:
          return True
    return False

  def compute_walkabout_strengths(self, cn):
    current_outputs = cn.selected_method.outputs
    for out_var in current_outputs:
      min_strength = cn.strength
      for mt in cn.methods:
        if not out_var in mt.outputs:
          max_strength = self.max_out(mt, current_outputs)
          if Strength.weaker(max_strength, min_strength):
            min_strength = max_strength
      out_var.walk_strength = min_strength

  def collect_unenforced(self, unenforced_cns, vars, collection_strength, collect_equal_strength):
    self._new_mark()
    for var in vars:
      unenforced_cns.extend(self.collect_unenforced_mark(unenforced_cns, var, collection_strength, collect_equal_strength))
    return unenforced_cns

  def collect_unenforced_mark(self, unenforced_cns, var, collection_strength, collect_equal_strength):
    for cn in var.constraints:
      if cn != var.determined_by and cn.mark != self.mark:
        cn.mark = self.mark
        if cn.is_enforced():
          for out_var in cn.selected_method.outputs:
            self.collect_unenforced_mark(unenforced_cns, out_var, collection_strength, collect_equal_strength)
        elif Strength.weaker(cn.strength, collection_strength) or \
              (collect_equal_strength and (cn.strength == collection_strength)):
          unenforced_cns.append(cn)
    return unenforced_cns

  def exec_pplan_create(self, exec_roots):
    exec_pplan = []

    for cn in exec_roots:
      if isinstance(cn, Constraint):
        cn.add_to_pplan(exec_pplan, self.mark)

    for var in exec_roots:
      if isinstance(var, Variable):
        if var.determined_by == None and not var.valid:
          exec_pplan.extend(var.add_to_pplan([], self.mark))
          var.valid = True

    return exec_pplan

  def exec_from_roots(self, exec_roots):
    self._new_mark()
    exec_pplan = self.exec_pplan_create(exec_roots)

    while exec_pplan:
      cn = exec_pplan.pop()
      if cn.mark != self.mark:
        # this cn has already been processed: do nothing
        # WTF?
        continue
      elif self.any_immediate_upstream_marked(cn):
        self.exec_from_cycle(cn)
      else:
        cn.mark = None
        self.execute_propagate_valid(cn)

  def execute_propagate_valid(self, cn):
    inputs_valid = True
    for var in cn.selected_method.inputs:
      if not var.valid:
        inputs_valid = False
    if inputs_valid:
      cn.selected_method.execute()
    for var in cn.selected_method.outputs:
      var.valid = inputs_valid

  def exec_from_cycle(self, cn):
    if cn.mark == self.mark:
      cn.mark = None
      for var in cn.selected_method.outputs:
        var.valid = False
        for consuming_cn in var.constraints:
          if consuming_cn != cn and consuming_cn.is_enforced:
            self.exec_from_cycle(consuming_cn)

  def pplan_add(self, pplan, objs):
    if not isinstance(objs, list): raise Exception("accepting only list of objs!")
    for elt in objs:
      elt.add_to_pplan(pplan, self.mark)
    return pplan

  def _new_mark(self):
    self.marker.new_mark()

  @property
  def mark(self):
    return self.marker.mark

  # def extract_plan(self, root_cns):
  #   good_cns = []
  #   bad_cns = []
  #   self._new_mark()
  #   pplan = self.pplan_add([], root_cns)

  #   while pplan:
  #     cn = pplan.pop()
  #     if cn.mark != self.mark:
  #       pass
  #     elif self.any_immediate_upstream_marked(cn):
  #       bad_cns.append(cn)
  #     else:
  #       cn.mark = None
  #       good_cns.append(cn)
  #   return self.create_valid_plan(root_cns, good_cns, bad_cns)

  # def create_valid_plan(self, root_cns, good_cns, bad_cns):
  #   plan = Plan(root_cns, good_cns, bad_cns)
  #   for cns in [root_cns, good_cns, bad_cns]:
  #     for cn in cns:
  #       cn.valid_plans.append(plan)
  #   return plan

  # def invalidate_plans_on_setting_method(self, cn, new_mt):
  #   self.invalidate_constraint_plans(cl.valid_plans)
  #   if new_mt != None:
  #     for var in new_mt.inputs:
  #       if var.determined_by != None:
  #         self.invalidate_constraint_plans(var.determined_by)

  # def invalidate_constraint_plans(self, invalid_cn):
  #   for plan in invalid_cn.valid_plans:
  #     plan.valid = False
  #     for cns in [plan.good_cns, plan.bad_cns, plan.root_cns]:
  #       for cn in cns:
  #         if cn != invalid_cn:
  #           cn.valid_plans.remove(plan)
  #   invalid_cn.valid_plans.clear()

  # def execute_plan(self, plan):
  #   if plan.valid:
  #     for cn in plan.good_cns:
  #       self.execute_propagate_valid(cn)
  #   else:
  #     raise ValueError("trying to execute invalid plan")

  def max_out(self, mt, current_outputs):
    max_strength = Strength.WEAKEST
    for var in mt.outputs:
      if not var in current_outputs and \
        Strength.weaker(max_strength, var.walk_strength):
        max_strength = var.walk_strength
    return max_strength

<|endoftext|>


<commit_before>#
#
# Copyright (C) 2010-2011 Huang Xin
#
# See LICENSE.TXT that came with this file.

from StimControl.mangrating import FrameSweep
from VisionEgg.Gratings import SinGrating2D
grating = SinGrating2D()
import pickle
with open('test_pickle.pkl','w') as file:
    pickle.dump(grating,file,1)
with open('test_pickle.pkl','r') as file:
    unpckled_stim = pickle.load(file)

sweep = FrameSweep()
sweep.add_stimulus(unpckled_stim)

sweep.go()

<commit_msg>pickling whole stimulus object seems impossible<commit_after>#
#
# Copyright (C) 2010-2011 Huang Xin
#
# See LICENSE.TXT that came with this file.

from StimControl.LightStim.SweepSeque import ParamSeque
from StimControl.LightStim.FrameControl import FrameSweep
from StimControl.LightStim.Grating import ParamsGrating
from StimControl.LightStim.Core import DefaultScreen
from StimControl.LightStim.LightData import dictattr,IndexedParam

DefaultScreen(['left','right'])

p = dictattr()
p.ml = 0.5
p.tfreqCycSec = 2.0
p.bgbrightness = 0.0
p.phase0 = 0
p.contrast = 1

orientation = IndexedParam('orientation')
spatial_freq = [None]
phase_at_t0 = [None]

param_sequence = ParamSeque(repeat=4, orientation=orientation, spatial_freq=spatial_freq, phase_at_t0=phase_at_t0, frame_duration=2.0, blank_duration=1.0)

random_grating = ParamsGrating(viewport='left', params=p, sweepseq=param_sequence)

import pickle
with open('test_pickle.pkl','w') as file:
    pickle.dump(random_grating,file,1)
with open('test_pickle.pkl','r') as file:
    unpckled_stim = pickle.load(file)

sweep = FrameSweep()
sweep.add_stimulus(unpckled_stim)

sweep.go()

<|endoftext|>


<commit_before>import json
import csv
from flask import (Flask, redirect, render_template, request, url_for)

from .gazer import Gazer

app = Flask(__name__)

crystal_gazer = Gazer()

@app.route('/')
def index():
    return render_template("index.html")

@app.route('/healthcheck')
def healthcheck():
    return {
        'status': 200,
        'status_text': 'OK'
    }

@app.route('/get_prediction', methods=['POST'])
def get_prediction():
    csv_file = request.files["csvFile"]
    file_name = csv_file.filename
    verdict, csv_data = crystal_gazer.get_prediction_for_data(csv_file.stream)
    results = verdict.values.tolist()
    correct = 0
    for x in range(0, len(results)):
        if csv_data["Bootcamp"][x] == int(round(results[x][3])):
            correct = correct + 1

    accuracy = (correct / len(results)) * 100
    return render_template(
        "result.html",
        file_name=file_name,
        results=results,
        accuracy=round(accuracy, 2)
    )
<commit_msg>Fix division error<commit_after>import json
import csv
from flask import (Flask, redirect, render_template, request, url_for)

from .gazer import Gazer

app = Flask(__name__)

crystal_gazer = Gazer()

@app.route('/')
def index():
    return render_template("index.html")

@app.route('/healthcheck')
def healthcheck():
    return {
        'status': 200,
        'status_text': 'OK'
    }

@app.route('/get_prediction', methods=['POST'])
def get_prediction():
    csv_file = request.files["csvFile"]
    file_name = csv_file.filename
    verdict, csv_data = crystal_gazer.get_prediction_for_data(csv_file.stream)
    results = verdict.values.tolist()
    correct = 0
    for x in range(0, len(results)):
        if csv_data["Bootcamp"][x] == int(round(results[x][3])):
            correct = correct + 1

    accuracy = (float(correct) / float(len(results))) * 100
    return render_template(
        "result.html",
        file_name=file_name,
        results=results,
        accuracy=round(accuracy, 2)
    )
<|endoftext|>


<commit_before>import re

from streamlink.plugin import Plugin
from streamlink.plugin.api import http
from streamlink.stream import HLSStream

HLS_URL_FORMAT = "https://hls.goodgame.ru/hls/{0}{1}.m3u8"
QUALITIES = {
    "1080p": "",
    "720p": "_720",
    "480p": "_480",
    "240p": "_240"
}

_url_re = re.compile(r"https?://(?:www\.)?goodgame.ru/channel/(?P<user>\w+)")
_stream_re = re.compile(r'var src = "([^"]+)";')
_ddos_re = re.compile(r'document.cookie="(__DDOS_[^;]+)')

class GoodGame(Plugin):
    @classmethod
    def can_handle_url(cls, url):
        return _url_re.match(url)

    def _check_stream(self, url):
        res = http.get(url, acceptable_status=(200, 404))
        if res.status_code == 200:
            return True

    def _get_streams(self):
        headers = {
            "Referer": self.url
        }
        res = http.get(self.url, headers=headers)

        match = _ddos_re.search(res.text)
        if match:
            headers["Cookie"] = match.group(1)
            res = http.get(self.url, headers=headers)

        match = _stream_re.search(res.text)
        if not match:
            return

        stream_id = match.group(1)
        streams = {}
        for name, url_suffix in QUALITIES.items():
            url = HLS_URL_FORMAT.format(stream_id, url_suffix)
            if not self._check_stream(url):
                continue

            streams[name] = HLSStream(self.session, url)

        return streams

__plugin__ = GoodGame
<commit_msg>[goodgame] Fixed url regexp for handling miscellaneous symbols in username.<commit_after>import re

from streamlink.plugin import Plugin
from streamlink.plugin.api import http
from streamlink.stream import HLSStream

HLS_URL_FORMAT = "https://hls.goodgame.ru/hls/{0}{1}.m3u8"
QUALITIES = {
    "1080p": "",
    "720p": "_720",
    "480p": "_480",
    "240p": "_240"
}

_url_re = re.compile(r"https?://(?:www\.)?goodgame.ru/channel/(?P<user>[^/]+)")
_stream_re = re.compile(r'var src = "([^"]+)";')
_ddos_re = re.compile(r'document.cookie="(__DDOS_[^;]+)')

class GoodGame(Plugin):
    @classmethod
    def can_handle_url(cls, url):
        return _url_re.match(url)

    def _check_stream(self, url):
        res = http.get(url, acceptable_status=(200, 404))
        if res.status_code == 200:
            return True

    def _get_streams(self):
        headers = {
            "Referer": self.url
        }
        res = http.get(self.url, headers=headers)

        match = _ddos_re.search(res.text)
        if match:
            headers["Cookie"] = match.group(1)
            res = http.get(self.url, headers=headers)

        match = _stream_re.search(res.text)
        if not match:
            return

        stream_id = match.group(1)
        streams = {}
        for name, url_suffix in QUALITIES.items():
            url = HLS_URL_FORMAT.format(stream_id, url_suffix)
            if not self._check_stream(url):
                continue

            streams[name] = HLSStream(self.session, url)

        return streams

__plugin__ = GoodGame
<|endoftext|>


<commit_before>import time
import traceback
import threading
import collections

from synapse.compat import queue

import synapse.lib.sched as s_sched
import synapse.lib.queue as s_queue
import synapse.dyndeps as s_dyndeps
import synapse.session as s_session
import synapse.lib.threads as s_threads

from synapse.common import *
from synapse.eventbus import EventBus

def jobid():
    return guidstr()

def jobret(job):
    '''
    Return job result or raise on error.

    Example:

        return jobret(job)

    '''
    err = job[1].get('err')
    if err != None:
        raise JobErr(job)
    return job[1].get('ret')

def jobDoneMesg(job):
    '''
    Construct a job:done message for the given job.

    Example:

        def ondone(job):
            otherguy.dist( jobDoneMesg(job) )

    '''
    info = {'jid':job[0], 'ret':job[1].get('ret')}
    if job[1].get('err') != None:
        info['err'] = job[1].get('err'),
        info['errmsg'] = job[1].get('errmsg'),
        info['errfile'] = job[1].get('errfile'),
        info['errline'] = job[1].get('errline'),

    return tufo('job:done', **info)

def newtask(meth,*args,**kwargs):
    return (meth,args,kwargs)

class Boss(EventBus):
    '''
    A Boss manages asynchonous jobs.

    ( If given a pool, he will execute them as well )

    Additionally, if given a pool, the boss will assume
    he is responsible for calling fini() on the pool when
    he is notified of fini()
    '''
    def __init__(self):
        EventBus.__init__(self)

        self.onfini( self._onBossFini )

        self.pool = None
        self.sched = s_sched.getGlobSched()

        self.joblock = threading.Lock()

        self._boss_jobs = {}
        self.joblocal = {}

        self.on('job:done', self._onJobDone )  # trigger job done
        self.on('job:fini', self._onJobFini )  # job is finished

    def setBossPool(self, pool):
        '''
        Set a thread pool object for the boss.

        Example:

            pool = s_threads.Pool(size=3)
            boss.setBossPool(pool)

        '''
        self.pool = pool

    def runBossPool(self, size, maxsize=None):
        '''
        Create and run a thread pool for this Boss()

        Example:

            boss.runBossPool(3)

        '''
        pool = s_threads.Pool(size=size, maxsize=maxsize)
        self.onfini( pool.fini )
        self.setBossPool(pool)

    def _onJobDone(self, event):
        # used to *trigger* job done processing
        jid = event[1].get('jid')
        if jid == None:
            return

        job = self._boss_jobs.get(jid)
        if job == None:
            return

        job[1].update( event[1] )

        job[1]['done'] = True
        self.fire('job:fini', job=job)

    def jobs(self):
        '''
        Return a list of job tuples.

        Example:

            for job in boss.jobs():
                dostuff(job)

        '''
        return list(self._boss_jobs.values())

    def __iter__(self):
        return self.jobs()

    def job(self, jid):
        '''
        Return a job tuple by ID.

        Example:

            job = boss.job(jid)

        '''
        return self._boss_jobs.get(jid)

    def initJob(self, jid=None, **info):
        '''
        Initialize and return a new job tufo.

        Example:

            jid = jobid()
            task = (meth,args,kwargs)

            job = boss.initJob(jid,task=task,timeout=60)

        Notes:

            Info Conventions:
            * task=(meth,args,kwargs)
            * timeout=<sec> # max runtime
            * ondone=<meth> # def ondone(job):

        '''
        if self.isfini:
            raise IsFini()

        if jid == None:
            jid = guidstr()

        info['done'] = False
        info['times'] = []

        job = (jid,info)

        self._boss_jobs[jid] = job

        # setup our per job local storage
        # ( for non-serializables )
        joblocal = {
            'ondone':info.pop('ondone',None),
        }
        self.joblocal[jid] = joblocal

        self._addJobTime(job,'init')

        if self.pool != None:
            self.pool.call( self._runJob, job )

        # if we have a timeout, setup a sched callback
        timeout = job[1].get('timeout')
        if timeout != None:

            def hitmax():
                joblocal.pop('schedevt',None)
                self.fire('job:done', jid=jid, err='HitMaxTime')

            joblocal['schedevt'] = self.sched.insec(timeout,hitmax)

        return job

    def sync(self, job, timeout=None):
        '''
        Wait and return the value for the job.
        '''
        if not self.wait(job[0], timeout=timeout):
            raise MaxTimeHit(timeout)

        return jobret(job)

    def _runJob(self, job):
        '''
        Actually execute the given job with the caller thread.
        '''
        task = job[1].get('task')
        if task == None:
            self.setJobErr(job[0],'NoJobTask')
            return

        try:

            func,args,kwargs = task
            ret = func(*args,**kwargs)
            self.fire('job:done', jid=job[0], ret=ret)

        except Exception as e:
            self.fire('job:done', jid=job[0], **excinfo(e))

    def _onJobFini(self, event):

        with self.joblock:
            job = event[1].get('job')
            jid = job[0]

            self._boss_jobs.pop(jid,None)
            joblocal = self.joblocal.pop(jid,None)

            schedevt = joblocal.get('schedevt')
            if schedevt != None:
                self.sched.cancel(schedevt)

            evt = joblocal.get('waitevt')
            if evt != None:
                evt.set()

            ondone = joblocal.get('ondone')
            if ondone != None:
                try:
                    ondone(job)
                except Exception as e:
                    traceback.print_exc()

    def wait(self, jid, timeout=None):
        '''
        Wait for a job to complete by id.

        Example:

            boss.wait( jid, timeout=10 )

        '''
        with self.joblock:
            job = self._boss_jobs.get(jid)
            if job == None:
                return True

            joblocal = self.joblocal.get(jid)
            evt = joblocal.get('waitevt')
            if evt == None:
                evt = threading.Event()
                joblocal['waitevt'] = evt

        evt.wait(timeout=timeout)
        return evt.is_set()

    def _addJobTime(self, job, stage):
        job[1]['times'].append( (stage,time.time()) )

    def _onBossFini(self):
        for job in self.jobs():
            self.fire('job:done', jid=job[0], err='BossShutDown')
<commit_msg>added job:init event to Boss<commit_after>import time
import traceback
import threading
import collections

from synapse.compat import queue

import synapse.lib.sched as s_sched
import synapse.lib.queue as s_queue
import synapse.dyndeps as s_dyndeps
import synapse.session as s_session
import synapse.lib.threads as s_threads

from synapse.common import *
from synapse.eventbus import EventBus

def jobid():
    return guidstr()

def jobret(job):
    '''
    Return job result or raise on error.

    Example:

        return jobret(job)

    '''
    err = job[1].get('err')
    if err != None:
        raise JobErr(job)
    return job[1].get('ret')

def jobDoneMesg(job):
    '''
    Construct a job:done message for the given job.

    Example:

        def ondone(job):
            otherguy.dist( jobDoneMesg(job) )

    '''
    info = {'jid':job[0], 'ret':job[1].get('ret')}
    if job[1].get('err') != None:
        info['err'] = job[1].get('err'),
        info['errmsg'] = job[1].get('errmsg'),
        info['errfile'] = job[1].get('errfile'),
        info['errline'] = job[1].get('errline'),

    return tufo('job:done', **info)

def newtask(meth,*args,**kwargs):
    return (meth,args,kwargs)

class Boss(EventBus):
    '''
    A Boss manages asynchonous jobs.

    ( If given a pool, he will execute them as well )

    Additionally, if given a pool, the boss will assume
    he is responsible for calling fini() on the pool when
    he is notified of fini()
    '''
    def __init__(self):
        EventBus.__init__(self)

        self.onfini( self._onBossFini )

        self.pool = None
        self.sched = s_sched.getGlobSched()

        self.joblock = threading.Lock()

        self._boss_jobs = {}
        self.joblocal = {}

        self.on('job:done', self._onJobDone )  # trigger job done
        self.on('job:fini', self._onJobFini )  # job is finished

    def setBossPool(self, pool):
        '''
        Set a thread pool object for the boss.

        Example:

            pool = s_threads.Pool(size=3)
            boss.setBossPool(pool)

        '''
        self.pool = pool

    def runBossPool(self, size, maxsize=None):
        '''
        Create and run a thread pool for this Boss()

        Example:

            boss.runBossPool(3)

        '''
        pool = s_threads.Pool(size=size, maxsize=maxsize)
        self.onfini( pool.fini )
        self.setBossPool(pool)

    def _onJobDone(self, event):
        # used to *trigger* job done processing
        jid = event[1].get('jid')
        if jid == None:
            return

        job = self._boss_jobs.get(jid)
        if job == None:
            return

        job[1].update( event[1] )

        job[1]['done'] = True
        self.fire('job:fini', job=job)

    def jobs(self):
        '''
        Return a list of job tuples.

        Example:

            for job in boss.jobs():
                dostuff(job)

        '''
        return list(self._boss_jobs.values())

    def __iter__(self):
        return self.jobs()

    def job(self, jid):
        '''
        Return a job tuple by ID.

        Example:

            job = boss.job(jid)

        '''
        return self._boss_jobs.get(jid)

    def initJob(self, jid=None, **info):
        '''
        Initialize and return a new job tufo.

        Example:

            jid = jobid()
            task = (meth,args,kwargs)

            job = boss.initJob(jid,task=task,timeout=60)

        Notes:

            Info Conventions:
            * task=(meth,args,kwargs)
            * timeout=<sec> # max runtime
            * ondone=<meth> # def ondone(job):

        '''
        if self.isfini:
            raise IsFini()

        if jid == None:
            jid = guidstr()

        info['done'] = False
        info['times'] = []

        job = (jid,info)

        self._boss_jobs[jid] = job

        # setup our per job local storage
        # ( for non-serializables )
        joblocal = {
            'ondone':info.pop('ondone',None),
        }
        self.joblocal[jid] = joblocal

        self._addJobTime(job,'init')

        if self.pool != None:
            self.pool.call( self._runJob, job )

        # if we have a timeout, setup a sched callback
        timeout = job[1].get('timeout')
        if timeout != None:

            def hitmax():
                joblocal.pop('schedevt',None)
                self.fire('job:done', jid=jid, err='HitMaxTime')

            joblocal['schedevt'] = self.sched.insec(timeout,hitmax)

        self.fire('job:init', job=job)
        return job

    def sync(self, job, timeout=None):
        '''
        Wait and return the value for the job.
        '''
        if not self.wait(job[0], timeout=timeout):
            raise MaxTimeHit(timeout)

        return jobret(job)

    def _runJob(self, job):
        '''
        Actually execute the given job with the caller thread.
        '''
        task = job[1].get('task')
        if task == None:
            self.setJobErr(job[0],'NoJobTask')
            return

        try:

            func,args,kwargs = task
            ret = func(*args,**kwargs)
            self.fire('job:done', jid=job[0], ret=ret)

        except Exception as e:
            self.fire('job:done', jid=job[0], **excinfo(e))

    def _onJobFini(self, event):

        with self.joblock:
            job = event[1].get('job')
            jid = job[0]

            self._boss_jobs.pop(jid,None)
            joblocal = self.joblocal.pop(jid,None)

            schedevt = joblocal.get('schedevt')
            if schedevt != None:
                self.sched.cancel(schedevt)

            evt = joblocal.get('waitevt')
            if evt != None:
                evt.set()

            ondone = joblocal.get('ondone')
            if ondone != None:
                try:
                    ondone(job)
                except Exception as e:
                    traceback.print_exc()

    def wait(self, jid, timeout=None):
        '''
        Wait for a job to complete by id.

        Example:

            boss.wait( jid, timeout=10 )

        '''
        with self.joblock:
            job = self._boss_jobs.get(jid)
            if job == None:
                return True

            joblocal = self.joblocal.get(jid)
            evt = joblocal.get('waitevt')
            if evt == None:
                evt = threading.Event()
                joblocal['waitevt'] = evt

        evt.wait(timeout=timeout)
        return evt.is_set()

    def _addJobTime(self, job, stage):
        job[1]['times'].append( (stage,time.time()) )

    def _onBossFini(self):
        for job in self.jobs():
            self.fire('job:done', jid=job[0], err='BossShutDown')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.api.errors import SynapseError

from collections import namedtuple

Requester = namedtuple("Requester", [
    "user", "access_token_id", "is_guest", "device_id", "app_service",
])
"""
Represents the user making a request

Attributes:
    user (UserID):  id of the user making the request
    access_token_id (int|None):  *ID* of the access token used for this
        request, or None if it came via the appservice API or similar
    is_guest (bool):  True if the user making this request is a guest user
    device_id (str|None):  device_id which was set at authentication time
    app_service (ApplicationService|None):  the AS requesting on behalf of the user
"""

def create_requester(user_id, access_token_id=None, is_guest=False,
                     device_id=None, app_service=None):
    """
    Create a new ``Requester`` object

    Args:
        user_id (str|UserID):  id of the user making the request
        access_token_id (int|None):  *ID* of the access token used for this
            request, or None if it came via the appservice API or similar
        is_guest (bool):  True if the user making this request is a guest user
        device_id (str|None):  device_id which was set at authentication time
        app_service (ApplicationService|None):  the AS requesting on behalf of the user

    Returns:
        Requester
    """
    if not isinstance(user_id, UserID):
        user_id = UserID.from_string(user_id)
    return Requester(user_id, access_token_id, is_guest, device_id, app_service)

def get_domain_from_id(string):
    try:
        return string.split(":", 1)[1]
    except IndexError:
        raise SynapseError(400, "Invalid ID: %r" % (string,))

class DomainSpecificString(
        namedtuple("DomainSpecificString", ("localpart", "domain"))
):
    """Common base class among ID/name strings that have a local part and a
    domain name, prefixed with a sigil.

    Has the fields:

        'localpart' : The local part of the name (without the leading sigil)
        'domain' : The domain part of the name
    """

    # Deny iteration because it will bite you if you try to create a singleton
    # set by:
    #    users = set(user)
    def __iter__(self):
        raise ValueError("Attempted to iterate a %s" % (type(self).__name__,))

    # Because this class is a namedtuple of strings and booleans, it is deeply
    # immutable.
    def __copy__(self):
        return self

    def __deepcopy__(self, memo):
        return self

    @classmethod
    def from_string(cls, s):
        """Parse the string given by 's' into a structure object."""
        if len(s) < 1 or s[0] != cls.SIGIL:
            raise SynapseError(400, "Expected %s string to start with '%s'" % (
                cls.__name__, cls.SIGIL,
            ))

        parts = s[1:].split(':', 1)
        if len(parts) != 2:
            raise SynapseError(
                400, "Expected %s of the form '%slocalname:domain'" % (
                    cls.__name__, cls.SIGIL,
                )
            )

        domain = parts[1]

        # This code will need changing if we want to support multiple domain
        # names on one HS
        return cls(localpart=parts[0], domain=domain)

    def to_string(self):
        """Return a string encoding the fields of the structure object."""
        return "%s%s:%s" % (self.SIGIL, self.localpart, self.domain)

    @classmethod
    def is_valid(cls, s):
        try:
            cls.from_string(s)
            return True
        except:
            return False

    __str__ = to_string

    @classmethod
    def create(cls, localpart, domain,):
        return cls(localpart=localpart, domain=domain)

class UserID(DomainSpecificString):
    """Structure representing a user ID."""
    SIGIL = "@"

class RoomAlias(DomainSpecificString):
    """Structure representing a room name."""
    SIGIL = "#"

class RoomID(DomainSpecificString):
    """Structure representing a room id. """
    SIGIL = "!"

class EventID(DomainSpecificString):
    """Structure representing an event id. """
    SIGIL = "$"

class StreamToken(
    namedtuple("Token", (
        "room_key",
        "presence_key",
        "typing_key",
        "receipt_key",
        "account_data_key",
        "push_rules_key",
        "to_device_key",
        "device_list_key",
    ))
):
    _SEPARATOR = "_"

    @classmethod
    def from_string(cls, string):
        try:
            keys = string.split(cls._SEPARATOR)
            while len(keys) < len(cls._fields):
                # i.e. old token from before receipt_key
                keys.append("0")
            return cls(*keys)
        except:
            raise SynapseError(400, "Invalid Token")

    def to_string(self):
        return self._SEPARATOR.join([str(k) for k in self])

    @property
    def room_stream_id(self):
        # TODO(markjh): Awful hack to work around hacks in the presence tests
        # which assume that the keys are integers.
        if type(self.room_key) is int:
            return self.room_key
        else:
            return int(self.room_key[1:].split("-")[-1])

    def is_after(self, other):
        """Does this token contain events that the other doesn't?"""
        return (
            (other.room_stream_id < self.room_stream_id)
            or (int(other.presence_key) < int(self.presence_key))
            or (int(other.typing_key) < int(self.typing_key))
            or (int(other.receipt_key) < int(self.receipt_key))
            or (int(other.account_data_key) < int(self.account_data_key))
            or (int(other.push_rules_key) < int(self.push_rules_key))
            or (int(other.to_device_key) < int(self.to_device_key))
            or (int(other.device_list_key) < int(self.device_list_key))
        )

    def copy_and_advance(self, key, new_value):
        """Advance the given key in the token to a new value if and only if the
        new value is after the old value.
        """
        new_token = self.copy_and_replace(key, new_value)
        if key == "room_key":
            new_id = new_token.room_stream_id
            old_id = self.room_stream_id
        else:
            new_id = int(getattr(new_token, key))
            old_id = int(getattr(self, key))
        if old_id < new_id:
            return new_token
        else:
            return self

    def copy_and_replace(self, key, new_value):
        d = self._asdict()
        d[key] = new_value
        return StreamToken(**d)

StreamToken.START = StreamToken(
    *(["s0"] + ["0"] * (len(StreamToken._fields) - 1))
)

class RoomStreamToken(namedtuple("_StreamToken", "topological stream")):
    """Tokens are positions between events. The token "s1" comes after event 1.

            s0    s1
            |     |
        [0] V [1] V [2]

    Tokens can either be a point in the live event stream or a cursor going
    through historic events.

    When traversing the live event stream events are ordered by when they
    arrived at the homeserver.

    When traversing historic events the events are ordered by their depth in
    the event graph "topological_ordering" and then by when they arrived at the
    homeserver "stream_ordering".

    Live tokens start with an "s" followed by the "stream_ordering" id of the
    event it comes after. Historic tokens start with a "t" followed by the
    "topological_ordering" id of the event it comes after, followed by "-",
    followed by the "stream_ordering" id of the event it comes after.
    """
    __slots__ = []

    @classmethod
    def parse(cls, string):
        try:
            if string[0] == 's':
                return cls(topological=None, stream=int(string[1:]))
            if string[0] == 't':
                parts = string[1:].split('-', 1)
                return cls(topological=int(parts[0]), stream=int(parts[1]))
        except:
            pass
        raise SynapseError(400, "Invalid token %r" % (string,))

    @classmethod
    def parse_stream_token(cls, string):
        try:
            if string[0] == 's':
                return cls(topological=None, stream=int(string[1:]))
        except:
            pass
        raise SynapseError(400, "Invalid token %r" % (string,))

    def __str__(self):
        if self.topological is not None:
            return "t%d-%d" % (self.topological, self.stream)
        else:
            return "s%d" % (self.stream,)

class ThirdPartyInstanceID(
        namedtuple("ThirdPartyInstanceID", ("appservice_id", "network_id"))
):
    # Deny iteration because it will bite you if you try to create a singleton
    # set by:
    #    users = set(user)
    def __iter__(self):
        raise ValueError("Attempted to iterate a %s" % (type(self).__name__,))

    # Because this class is a namedtuple of strings, it is deeply immutable.
    def __copy__(self):
        return self

    def __deepcopy__(self, memo):
        return self

    @classmethod
    def from_string(cls, s):
        bits = s.split("|", 2)
        if len(bits) != 2:
            raise SynapseError(400, "Invalid ID %r" % (s,))

        return cls(appservice_id=bits[0], network_id=bits[1])

    def to_string(self):
        return "%s|%s" % (self.appservice_id, self.network_id,)

    __str__ = to_string

    @classmethod
    def create(cls, appservice_id, network_id,):
        return cls(appservice_id=appservice_id, network_id=network_id)
<commit_msg>Speed up copy_and_replace<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.api.errors import SynapseError

from collections import namedtuple

Requester = namedtuple("Requester", [
    "user", "access_token_id", "is_guest", "device_id", "app_service",
])
"""
Represents the user making a request

Attributes:
    user (UserID):  id of the user making the request
    access_token_id (int|None):  *ID* of the access token used for this
        request, or None if it came via the appservice API or similar
    is_guest (bool):  True if the user making this request is a guest user
    device_id (str|None):  device_id which was set at authentication time
    app_service (ApplicationService|None):  the AS requesting on behalf of the user
"""

def create_requester(user_id, access_token_id=None, is_guest=False,
                     device_id=None, app_service=None):
    """
    Create a new ``Requester`` object

    Args:
        user_id (str|UserID):  id of the user making the request
        access_token_id (int|None):  *ID* of the access token used for this
            request, or None if it came via the appservice API or similar
        is_guest (bool):  True if the user making this request is a guest user
        device_id (str|None):  device_id which was set at authentication time
        app_service (ApplicationService|None):  the AS requesting on behalf of the user

    Returns:
        Requester
    """
    if not isinstance(user_id, UserID):
        user_id = UserID.from_string(user_id)
    return Requester(user_id, access_token_id, is_guest, device_id, app_service)

def get_domain_from_id(string):
    try:
        return string.split(":", 1)[1]
    except IndexError:
        raise SynapseError(400, "Invalid ID: %r" % (string,))

class DomainSpecificString(
        namedtuple("DomainSpecificString", ("localpart", "domain"))
):
    """Common base class among ID/name strings that have a local part and a
    domain name, prefixed with a sigil.

    Has the fields:

        'localpart' : The local part of the name (without the leading sigil)
        'domain' : The domain part of the name
    """

    # Deny iteration because it will bite you if you try to create a singleton
    # set by:
    #    users = set(user)
    def __iter__(self):
        raise ValueError("Attempted to iterate a %s" % (type(self).__name__,))

    # Because this class is a namedtuple of strings and booleans, it is deeply
    # immutable.
    def __copy__(self):
        return self

    def __deepcopy__(self, memo):
        return self

    @classmethod
    def from_string(cls, s):
        """Parse the string given by 's' into a structure object."""
        if len(s) < 1 or s[0] != cls.SIGIL:
            raise SynapseError(400, "Expected %s string to start with '%s'" % (
                cls.__name__, cls.SIGIL,
            ))

        parts = s[1:].split(':', 1)
        if len(parts) != 2:
            raise SynapseError(
                400, "Expected %s of the form '%slocalname:domain'" % (
                    cls.__name__, cls.SIGIL,
                )
            )

        domain = parts[1]

        # This code will need changing if we want to support multiple domain
        # names on one HS
        return cls(localpart=parts[0], domain=domain)

    def to_string(self):
        """Return a string encoding the fields of the structure object."""
        return "%s%s:%s" % (self.SIGIL, self.localpart, self.domain)

    @classmethod
    def is_valid(cls, s):
        try:
            cls.from_string(s)
            return True
        except:
            return False

    __str__ = to_string

    @classmethod
    def create(cls, localpart, domain,):
        return cls(localpart=localpart, domain=domain)

class UserID(DomainSpecificString):
    """Structure representing a user ID."""
    SIGIL = "@"

class RoomAlias(DomainSpecificString):
    """Structure representing a room name."""
    SIGIL = "#"

class RoomID(DomainSpecificString):
    """Structure representing a room id. """
    SIGIL = "!"

class EventID(DomainSpecificString):
    """Structure representing an event id. """
    SIGIL = "$"

class StreamToken(
    namedtuple("Token", (
        "room_key",
        "presence_key",
        "typing_key",
        "receipt_key",
        "account_data_key",
        "push_rules_key",
        "to_device_key",
        "device_list_key",
    ))
):
    _SEPARATOR = "_"

    @classmethod
    def from_string(cls, string):
        try:
            keys = string.split(cls._SEPARATOR)
            while len(keys) < len(cls._fields):
                # i.e. old token from before receipt_key
                keys.append("0")
            return cls(*keys)
        except:
            raise SynapseError(400, "Invalid Token")

    def to_string(self):
        return self._SEPARATOR.join([str(k) for k in self])

    @property
    def room_stream_id(self):
        # TODO(markjh): Awful hack to work around hacks in the presence tests
        # which assume that the keys are integers.
        if type(self.room_key) is int:
            return self.room_key
        else:
            return int(self.room_key[1:].split("-")[-1])

    def is_after(self, other):
        """Does this token contain events that the other doesn't?"""
        return (
            (other.room_stream_id < self.room_stream_id)
            or (int(other.presence_key) < int(self.presence_key))
            or (int(other.typing_key) < int(self.typing_key))
            or (int(other.receipt_key) < int(self.receipt_key))
            or (int(other.account_data_key) < int(self.account_data_key))
            or (int(other.push_rules_key) < int(self.push_rules_key))
            or (int(other.to_device_key) < int(self.to_device_key))
            or (int(other.device_list_key) < int(self.device_list_key))
        )

    def copy_and_advance(self, key, new_value):
        """Advance the given key in the token to a new value if and only if the
        new value is after the old value.
        """
        new_token = self.copy_and_replace(key, new_value)
        if key == "room_key":
            new_id = new_token.room_stream_id
            old_id = self.room_stream_id
        else:
            new_id = int(getattr(new_token, key))
            old_id = int(getattr(self, key))
        if old_id < new_id:
            return new_token
        else:
            return self

    def copy_and_replace(self, key, new_value):
        return self._replace(**{key: new_value})

StreamToken.START = StreamToken(
    *(["s0"] + ["0"] * (len(StreamToken._fields) - 1))
)

class RoomStreamToken(namedtuple("_StreamToken", "topological stream")):
    """Tokens are positions between events. The token "s1" comes after event 1.

            s0    s1
            |     |
        [0] V [1] V [2]

    Tokens can either be a point in the live event stream or a cursor going
    through historic events.

    When traversing the live event stream events are ordered by when they
    arrived at the homeserver.

    When traversing historic events the events are ordered by their depth in
    the event graph "topological_ordering" and then by when they arrived at the
    homeserver "stream_ordering".

    Live tokens start with an "s" followed by the "stream_ordering" id of the
    event it comes after. Historic tokens start with a "t" followed by the
    "topological_ordering" id of the event it comes after, followed by "-",
    followed by the "stream_ordering" id of the event it comes after.
    """
    __slots__ = []

    @classmethod
    def parse(cls, string):
        try:
            if string[0] == 's':
                return cls(topological=None, stream=int(string[1:]))
            if string[0] == 't':
                parts = string[1:].split('-', 1)
                return cls(topological=int(parts[0]), stream=int(parts[1]))
        except:
            pass
        raise SynapseError(400, "Invalid token %r" % (string,))

    @classmethod
    def parse_stream_token(cls, string):
        try:
            if string[0] == 's':
                return cls(topological=None, stream=int(string[1:]))
        except:
            pass
        raise SynapseError(400, "Invalid token %r" % (string,))

    def __str__(self):
        if self.topological is not None:
            return "t%d-%d" % (self.topological, self.stream)
        else:
            return "s%d" % (self.stream,)

class ThirdPartyInstanceID(
        namedtuple("ThirdPartyInstanceID", ("appservice_id", "network_id"))
):
    # Deny iteration because it will bite you if you try to create a singleton
    # set by:
    #    users = set(user)
    def __iter__(self):
        raise ValueError("Attempted to iterate a %s" % (type(self).__name__,))

    # Because this class is a namedtuple of strings, it is deeply immutable.
    def __copy__(self):
        return self

    def __deepcopy__(self, memo):
        return self

    @classmethod
    def from_string(cls, s):
        bits = s.split("|", 2)
        if len(bits) != 2:
            raise SynapseError(400, "Invalid ID %r" % (s,))

        return cls(appservice_id=bits[0], network_id=bits[1])

    def to_string(self):
        return "%s|%s" % (self.appservice_id, self.network_id,)

    __str__ = to_string

    @classmethod
    def create(cls, appservice_id, network_id,):
        return cls(appservice_id=appservice_id, network_id=network_id)
<|endoftext|>


<commit_before># Copyright 2020, The TensorFlow Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Tests for tensorflow_privacy.privacy.membership_inference_attack.data_structures."""
from absl.testing import absltest
import numpy as np
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackInputData
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackResults
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackType
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import RocCurve
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import SingleAttackResult

class AttackInputDataTest(absltest.TestCase):

  def test_get_loss(self):
    attack_input = AttackInputData(
        logits_train=np.array([[0.3, 0.5, 0.2], [0.2, 0.3, 0.5]]),
        logits_test=np.array([[0.2, 0.3, 0.5], [0.3, 0.5, 0.2]]),
        labels_train=np.array([1, 0]),
        labels_test=np.array([0, 1]))

    np.testing.assert_equal(attack_input.get_loss_train().tolist(), [0.5, 0.2])
    np.testing.assert_equal(attack_input.get_loss_test().tolist(), [0.2, 0.5])

  def test_get_loss_explicitly_provided(self):
    attack_input = AttackInputData(
        loss_train=np.array([1.0, 3.0, 6.0]),
        loss_test=np.array([1.0, 4.0, 6.0]))

    np.testing.assert_equal(attack_input.get_loss_train().tolist(),
                            [1.0, 3.0, 6.0])
    np.testing.assert_equal(attack_input.get_loss_test().tolist(),
                            [1.0, 4.0, 6.0])

  def test_validator(self):
    self.assertRaises(ValueError,
                      AttackInputData(logits_train=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(labels_train=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(loss_train=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(logits_test=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(labels_test=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(loss_test=np.array([])).validate)
    self.assertRaises(ValueError, AttackInputData().validate)

class RocCurveTest(absltest.TestCase):

  def test_auc_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_auc(), 0.5)

  def test_auc_perfect_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 1.0, 1.0]),
        fpr=np.array([1.0, 1.0, 0.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_auc(), 1.0)

  def test_attacker_advantage_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_attacker_advantage(), 0.0)

  def test_attacker_advantage_perfect_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 1.0, 1.0]),
        fpr=np.array([1.0, 1.0, 0.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_auc(), 1.0)

class SingleAttackResultTest(absltest.TestCase):

  # Only a basic test, as this method calls RocCurve which is tested separately.
  def test_auc_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    result = SingleAttackResult(
        roc_curve=roc, slice_spec=None, attack_type=AttackType.THRESHOLD_ATTACK)

    self.assertEqual(result.get_auc(), 0.5)

  # Only a basic test, as this method calls RocCurve which is tested separately.
  def test_attacker_advantage_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    result = SingleAttackResult(
        roc_curve=roc, slice_spec=None, attack_type=AttackType.THRESHOLD_ATTACK)

    self.assertEqual(result.get_attacker_advantage(), 0.0)

class AttackResultsTest(absltest.TestCase):

  perfect_classifier_result: SingleAttackResult
  random_classifier_result: SingleAttackResult

  def __init__(self, *args, **kwargs):
    super(AttackResultsTest, self).__init__(*args, **kwargs)

    # ROC curve of a perfect classifier
    self.perfect_classifier_result = SingleAttackResult(
        slice_spec=None,
        attack_type=AttackType.THRESHOLD_ATTACK,
        roc_curve=RocCurve(
            tpr=np.array([0.0, 1.0, 1.0]),
            fpr=np.array([1.0, 1.0, 0.0]),
            thresholds=np.array([0, 1, 2])))

    # ROC curve of a random classifier
    self.random_classifier_result = SingleAttackResult(
        slice_spec=None,
        attack_type=AttackType.THRESHOLD_ATTACK,
        roc_curve=RocCurve(
            tpr=np.array([0.0, 0.5, 1.0]),
            fpr=np.array([0.0, 0.5, 1.0]),
            thresholds=np.array([0, 1, 2])))

  def test_get_result_with_max_auc_first(self):
    results = AttackResults(
        [self.perfect_classifier_result, self.random_classifier_result])
    self.assertEqual(results.get_result_with_max_auc(),
                     self.perfect_classifier_result)

  def test_get_result_with_max_auc_second(self):
    results = AttackResults(
        [self.random_classifier_result, self.perfect_classifier_result])
    self.assertEqual(results.get_result_with_max_auc(),
                     self.perfect_classifier_result)

  def test_get_result_with_max_attacker_advantage_first(self):
    results = AttackResults(
        [self.perfect_classifier_result, self.random_classifier_result])
    self.assertEqual(results.get_result_with_max_attacker_advantage(),
                     self.perfect_classifier_result)

  def test_get_result_with_max_attacker_advantage_second(self):
    results = AttackResults(
        [self.random_classifier_result, self.perfect_classifier_result])
    self.assertEqual(results.get_result_with_max_attacker_advantage(),
                     self.perfect_classifier_result)

if __name__ == '__main__':
  absltest.main()
<commit_msg>  Internal change.<commit_after># Copyright 2020, The TensorFlow Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Tests for tensorflow_privacy.privacy.membership_inference_attack.data_structures."""
from absl.testing import absltest
import numpy as np
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackInputData
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackResults
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackType
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import RocCurve
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import SingleAttackResult
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import SingleSliceSpec
from tensorflow_privacy.privacy.membership_inference_attack.data_structures import SlicingFeature

class AttackInputDataTest(absltest.TestCase):

  def test_get_loss(self):
    attack_input = AttackInputData(
        logits_train=np.array([[0.3, 0.5, 0.2], [0.2, 0.3, 0.5]]),
        logits_test=np.array([[0.2, 0.3, 0.5], [0.3, 0.5, 0.2]]),
        labels_train=np.array([1, 0]),
        labels_test=np.array([0, 1]))

    np.testing.assert_equal(attack_input.get_loss_train().tolist(), [0.5, 0.2])
    np.testing.assert_equal(attack_input.get_loss_test().tolist(), [0.2, 0.5])

  def test_get_loss_explicitly_provided(self):
    attack_input = AttackInputData(
        loss_train=np.array([1.0, 3.0, 6.0]),
        loss_test=np.array([1.0, 4.0, 6.0]))

    np.testing.assert_equal(attack_input.get_loss_train().tolist(),
                            [1.0, 3.0, 6.0])
    np.testing.assert_equal(attack_input.get_loss_test().tolist(),
                            [1.0, 4.0, 6.0])

  def test_validator(self):
    self.assertRaises(ValueError,
                      AttackInputData(logits_train=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(labels_train=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(loss_train=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(logits_test=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(labels_test=np.array([])).validate)
    self.assertRaises(ValueError,
                      AttackInputData(loss_test=np.array([])).validate)
    self.assertRaises(ValueError, AttackInputData().validate)

class RocCurveTest(absltest.TestCase):

  def test_auc_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_auc(), 0.5)

  def test_auc_perfect_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 1.0, 1.0]),
        fpr=np.array([1.0, 1.0, 0.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_auc(), 1.0)

  def test_attacker_advantage_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_attacker_advantage(), 0.0)

  def test_attacker_advantage_perfect_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 1.0, 1.0]),
        fpr=np.array([1.0, 1.0, 0.0]),
        thresholds=np.array([0, 1, 2]))

    self.assertEqual(roc.get_auc(), 1.0)

class SingleAttackResultTest(absltest.TestCase):

  # Only a basic test, as this method calls RocCurve which is tested separately.
  def test_auc_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    result = SingleAttackResult(
        roc_curve=roc,
        slice_spec=SingleSliceSpec(None),
        attack_type=AttackType.THRESHOLD_ATTACK)

    self.assertEqual(result.get_auc(), 0.5)

  # Only a basic test, as this method calls RocCurve which is tested separately.
  def test_attacker_advantage_random_classifier(self):
    roc = RocCurve(
        tpr=np.array([0.0, 0.5, 1.0]),
        fpr=np.array([0.0, 0.5, 1.0]),
        thresholds=np.array([0, 1, 2]))

    result = SingleAttackResult(
        roc_curve=roc,
        slice_spec=SingleSliceSpec(None),
        attack_type=AttackType.THRESHOLD_ATTACK)

    self.assertEqual(result.get_attacker_advantage(), 0.0)

class AttackResultsTest(absltest.TestCase):

  perfect_classifier_result: SingleAttackResult
  random_classifier_result: SingleAttackResult

  def __init__(self, *args, **kwargs):
    super(AttackResultsTest, self).__init__(*args, **kwargs)

    # ROC curve of a perfect classifier
    self.perfect_classifier_result = SingleAttackResult(
        slice_spec=SingleSliceSpec(SlicingFeature.CORRECTLY_CLASSIFIED, True),
        attack_type=AttackType.THRESHOLD_ATTACK,
        roc_curve=RocCurve(
            tpr=np.array([0.0, 1.0, 1.0]),
            fpr=np.array([1.0, 1.0, 0.0]),
            thresholds=np.array([0, 1, 2])))

    # ROC curve of a random classifier
    self.random_classifier_result = SingleAttackResult(
        slice_spec=SingleSliceSpec(None),
        attack_type=AttackType.THRESHOLD_ATTACK,
        roc_curve=RocCurve(
            tpr=np.array([0.0, 0.5, 1.0]),
            fpr=np.array([0.0, 0.5, 1.0]),
            thresholds=np.array([0, 1, 2])))

  def test_get_result_with_max_auc_first(self):
    results = AttackResults(
        [self.perfect_classifier_result, self.random_classifier_result])
    self.assertEqual(results.get_result_with_max_auc(),
                     self.perfect_classifier_result)

  def test_get_result_with_max_auc_second(self):
    results = AttackResults(
        [self.random_classifier_result, self.perfect_classifier_result])
    self.assertEqual(results.get_result_with_max_auc(),
                     self.perfect_classifier_result)

  def test_get_result_with_max_attacker_advantage_first(self):
    results = AttackResults(
        [self.perfect_classifier_result, self.random_classifier_result])
    self.assertEqual(results.get_result_with_max_attacker_advantage(),
                     self.perfect_classifier_result)

  def test_get_result_with_max_attacker_advantage_second(self):
    results = AttackResults(
        [self.random_classifier_result, self.perfect_classifier_result])
    self.assertEqual(results.get_result_with_max_attacker_advantage(),
                     self.perfect_classifier_result)

  def test_summary_by_slices(self):
    results = AttackResults(
        [self.perfect_classifier_result, self.random_classifier_result])
    self.assertEqual(
        results.summary(by_slices=True), 'Highest AUC on slice '
                                         'SingleSliceSpec' +
        '(SlicingFeature.CORRECTLY_CLASSIFIED=True) achieved by ' +
        'AttackType.THRESHOLD_ATTACK with an AUC of 1.0\n' +
        'Highest advantage on ' +
        'slice SingleSliceSpec(SlicingFeature.CORRECTLY_CLASSIFIED=True) ' +
        'achieved by AttackType.THRESHOLD_ATTACK with an advantage of 1.0\n' +
        'Highest AUC on slice SingleSliceSpec(Entire dataset) achieved ' +
        'by AttackType.THRESHOLD_ATTACK with an AUC of 0.5\n' +
        'Highest advantage on slice SingleSliceSpec(Entire dataset) achieved ' +
        'by AttackType.THRESHOLD_ATTACK with an advantage of 0.0')

  def test_summary_without_slices(self):
    results = AttackResults(
        [self.perfect_classifier_result, self.random_classifier_result])
    self.assertEqual(
        results.summary(by_slices=False),
        'Highest AUC on slice SingleSliceSpec(Entire dataset) achieved ' +
        'by AttackType.THRESHOLD_ATTACK with an AUC of 0.5\n' +
        'Highest advantage on slice SingleSliceSpec(Entire dataset) achieved ' +
        'by AttackType.THRESHOLD_ATTACK with an advantage of 0.0')

if __name__ == '__main__':
  absltest.main()
<|endoftext|>


<commit_before>from plumbum import cli, local
from actions import SyncRemote, AbortException, SyncError

class SyncHg(cli.Application):
    name = cli.SwitchAttr(
            ['n', '--name'],
            help='The name of the repository.'
                 'Uses the directory name by default'
            )

    def main(self, remote_host, local_path=None):
        if local_path:
            local_path = local.cwd / local_path
        else:
            local_path = local.cwd

        if not self.name:
            self.name = local_path.basename

        SyncRemote(remote_host, self.name, local_path)

def run():
    try:
        SyncHg.run()
    except AbortException:
        pass
    except SyncError as e:
        # TODO: Colour would be nice here..
        print "Error: {0}".format(e)
<commit_msg>Added config file loading & creation<commit_after>import os
from ConfigParser import ConfigParser, Error as ConfigParserError
from plumbum import cli, local
from clint import resources
from actions import SyncRemote, AbortException, SyncError

class SyncHg(cli.Application):
    ConfigFileName = 'config.ini'

    def __init__(self, *pargs):
        super(SyncHg, self).__init__(*pargs)
        self.config = ConfigParser()

    name = cli.SwitchAttr(
            ['n', '--name'],
            help='The name of the repository. '
                 'Uses the directory name by default'
            )

    @cli.switch(['-c', '--config'])
    def do_config(self):
        '''
        Runs through the initial configuration
        '''
        print "Running initial configuration"
        self._get_config(True)
        srcdir = default = ''
        if not self.config.has_section('config'):
            self.config.add_section('config')
        try:
            default = self.config.get('config', 'hgroot')
        except ConfigParserError:
            pass
        while not srcdir:
            srcdir = raw_input(
                    "Remote source directory? [{0}] ".format(default)
                    )
        self.config.set('config', 'hgroot', srcdir)
        if not os.path.exists(resources.user.path):
            os.makedirs(resources.user.path)
        self.config.write(resources.user.open(self.ConfigFileName, 'w'))
        pass

    def main(self, remote_host, local_path=None):
        self._get_config()
        if local_path:
            local_path = local.cwd / local_path
        else:
            local_path = local.cwd

        if not self.name:
            self.name = local_path.basename

        SyncRemote(remote_host, self.name, local_path)

    def _get_config(self, in_do_config=False):
        '''
        Reads the configuration
        '''
        resources.init('obmarg', 'synchg')
        self.config = ConfigParser()
        contents = resources.user.read(self.ConfigFileName)
        if not contents:
            if not in_do_config:
                print "Could not find config file"
                self.do_config()
        else:
            self.config.readfp(resources.user.open(self.ConfigFileName))

def run():
    try:
        SyncHg.run()
    except AbortException:
        pass
    except SyncError as e:
        # TODO: Colour would be nice here..
        print "Error: {0}".format(e)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Trimming of trailing white spaces.

Author-email: "Dietmar Winkler" <dietmarw at gmx de>

License: See UNLICENSE file

This script will recursively remove all trailing white spaces in all
text files in a given directory. Binary files and files residing in
'.svn' or '.git' directories are skipped.

It uses a binary tester based on the python magic implementation from
	Adam Hupp, http://hupp.org/adam/hg/python-magic

As a fallback (especially if libmagic is not available, like on Windows)
it acts only on files with a given file extension listed in 'extstring'.

"""

import os
import sys
import textwrap

# For Windows: list of file extensions to apply the script on
# (including the dot and separated by a comma)
extstring = ".mo,.mos,.c,.h,.cpp,.txt"
# convert the string object to a list object
listofexts  = extstring.split(",")

def usage(args):
	"""Help message on usage."""
	message = """
		Usage: %s [OPTIONS] <directory>

		 This script will recursively remove all trailing white spaces in all
		 text files in a given directory. Binary files and files residing in
		 '.svn' or '.git' directories are skipped.

		Note for Windows users:
		 If you have not libmagic installed the script will fallback to only
		 trim files with the following extensions: %s

		Options:
			-h, --help
				displays this help message
		""" % (os.path.split(args[0])[1],extstring,)
	print textwrap.dedent(message)

def unkownOption(args):
	"""Warning message for unknown options."""
	warning = """
		UNKNOWN OPTION: "%s"

		Use %s -h [--help] for usage instructions.
		""" % (args[1],args[0],)
	print textwrap.dedent(warning)

def detecttype(filepath):
	"""Detect the mime type of the text file."""
	try:
		import magic
		mime = magic.Magic(mime=True)
		type = mime.from_file(filepath)
		if "text/" in type:
			return "text"
		else:
			return type
	except (ImportError, TypeError):
		root, ext = os.path.splitext(filepath)
		if ext in listofexts:
			return "text"
		else:
			return "unknown"

def main(args):
	import getopt
	# Look for optional arguments:
	try:
		opts, dirnames = getopt.getopt(args[1:], "h", ["help"])
	# Unknown option is given trigger the display message:
	except getopt.GetoptError:
		unkownOption(args)
		sys.exit(0)
	# if no dir name is given print the usage message
	if not dirnames:
		usage(args)
		sys.exit(0)

	# If help option is given display help otherwise display warning:
	for opt, arg in opts:
		if opt in ("-h","--help"):
			usage(args)
			sys.exit(0)
		else:
			unkownOption(args)
			sys.exit(0)

	# Walk through the given path and call trim function for text files only:
	for path, dirs, files in os.walk(args[1]):
		for file in files:
			filepath = os.path.join(path, file)
			filetype = detecttype(filepath)
			if ".svn" in path or ".git" in path:
				print "skipping version control file: "+filepath
			elif filetype is "text":
				print "trimming " + filepath
				trimWhitespace(filepath)
			else:
				print "skipping file of type "+filetype+": "+filepath

def trimWhitespace(filepath):
	"""Trim trailing white spaces from a given filepath."""
	lines = []
	for line in open(filepath, "r"):
		lines.append(line.rstrip())
	f = open(filepath, "w")
	f.write("\n".join(lines) + "\n")
	f.close

if __name__ == "__main__":
	sys.exit(main(sys.argv))
<commit_msg>First attempt to exclude more version control files (might be a wee bit inefficient though)<commit_after>#!/usr/bin/env python
"""Trimming of trailing white spaces.

Author-email: "Dietmar Winkler" <dietmarw at gmx de>

License: See UNLICENSE file

This script will recursively remove all trailing white spaces in all
text files in a given directory. Binary files and files residing in
'.svn' or '.git' directories are skipped.

It uses a binary tester based on the python magic implementation from
	Adam Hupp, http://hupp.org/adam/hg/python-magic

As a fallback (especially if libmagic is not available, like on Windows)
it acts only on files with a given file extension listed in 'extstring'.

"""

import os
import sys
import textwrap

# For Windows: list of file extensions to apply the script on
# (including the dot and separated by a comma)
extstring = ".mo,.mos,.c,.h,.cpp,.txt"
# convert the string object to a list object
listofexts  = extstring.split(",")

def usage(args):
	"""Help message on usage."""
	message = """
		Usage: %s [OPTIONS] <directory>

		 This script will recursively remove all trailing white spaces in all
		 text files in a given directory. Binary files and files residing in
		 '.svn' or '.git' directories are skipped.

		Note for Windows users:
		 If you have not libmagic installed the script will fallback to only
		 trim files with the following extensions: %s

		Options:
			-h, --help
				displays this help message
		""" % (os.path.split(args[0])[1],extstring,)
	print textwrap.dedent(message)

def unkownOption(args):
	"""Warning message for unknown options."""
	warning = """
		UNKNOWN OPTION: "%s"

		Use %s -h [--help] for usage instructions.
		""" % (args[1],args[0],)
	print textwrap.dedent(warning)

def detecttype(filepath):
	"""Detect the mime type of the text file."""
	try:
		import magic
		mime = magic.Magic(mime=True)
		type = mime.from_file(filepath)
		if "text/" in type:
			return "text"
		else:
			return type
	except (ImportError, TypeError):
		root, ext = os.path.splitext(filepath)
		if ext in listofexts:
			return "text"
		else:
			return "unknown"

def main(args):
	import getopt
	# Look for optional arguments:
	try:
		opts, dirnames = getopt.getopt(args[1:], "h", ["help"])
	# Unknown option is given trigger the display message:
	except getopt.GetoptError:
		unkownOption(args)
		sys.exit(0)
	# if no dir name is given print the usage message
	if not dirnames:
		usage(args)
		sys.exit(0)

	# If help option is given display help otherwise display warning:
	for opt, arg in opts:
		if opt in ("-h","--help"):
			usage(args)
			sys.exit(0)
		else:
			unkownOption(args)
			sys.exit(0)

	# Walk through the given path and call trim function for text files only:
	for path, dirs, files in os.walk(args[1]):
		for file in files:
			filepath = os.path.join(path, file)
			filetype = detecttype(filepath)
			EXCLUDES = ['.cvs', '.svn', '.git', '.hg', '.bzr']
			if [dirname in EXCLUDES for dirname in path.split(os.sep)]:
				print "skipping version control file: "+filepath
			elif filetype is "text":
				print "trimming " + filepath
				trimWhitespace(filepath)
			else:
				print "skipping file of type "+filetype+": "+filepath

def trimWhitespace(filepath):
	"""Trim trailing white spaces from a given filepath."""
	lines = []
	for line in open(filepath, "r"):
		lines.append(line.rstrip())
	f = open(filepath, "w")
	f.write("\n".join(lines) + "\n")
	f.close

if __name__ == "__main__":
	sys.exit(main(sys.argv))
<|endoftext|>