<commit_before>#!/usr/bin/env python
# this only exists because sympy crashes IDAPython
# for general use sympy is much more complete

import traceback
import types
import copy
import operator
import random
import string
from memoize import Memoize

def collect(exp, fn):
  rv = set()

  def _collect(exp):
    if fn(exp):
      rv.add(exp)
    return exp

  exp.walk(collect)
  return rv

class _Symbolic(tuple):

  def match(self, other, valuestore=None):
    '''
    matches against a pattern, use wilds() to generate wilds

    Example:
      a,b = wilds('a b')
      val = WildsResults()

      if exp.match(a(b + 4), val):
        print val.a
        print val.b
    '''
    import match
    return match.match(self, other, valuestore)

  def __hash__(self):
    return hash(self.name)

  def simplify(self):
    import simplify
    return simplify.simplify(self)

  def walk(self, *fns):
    if len(fns) > 1:
      def _(exp):
        for f in fns:
          exp = f(exp)
        return exp
      return self.walk(_)

    exp = self
    fn = fns[0]
    if len(exp) == 1:
      oldexp = exp
      exp = fn(exp)
      while exp != oldexp:
        oldexp = exp
        exp = fn(exp)

    else:
      args = list(map(lambda x: x.walk(fn), exp.args))
      oldexp = self
      exp = fn(fn(exp[0])(*args))

      #while exp != oldexp:
      #  print '%s => %s' % (oldexp, exp)
      #  oldexp = exp
      #  exp = exp.walk(fn)

    return exp

  def _dump(self):
    return {
        'name': self.name,
        'id': id(self)
        }

  def substitute(self, subs):
    '''
    takes a dictionary of substitutions
    returns itself with substitutions made
    '''
    if self in subs:
      self = subs[self]

    return self

  def __eq__(self, other):
    #return type(self) == type(other) and self.name == other.name
    return id(self) == id(other)

  def __ne__(self, other):
    return not self.__eq__(other)

  def __getitem__(self, num):
    if num == 0:
      return self

    raise BaseException("Invalid index")

  def __len__(self):
    return 1

  # comparison operations notice we don't override __eq__
  def __gt__(self, obj):
    return Fn.GreaterThan(self, obj)

  def __ge__(self, obj):
    return Fn.GreaterThanEq(self, obj)

  def __lt__(self, obj):
    return Fn.LessThan(self, obj)

  def __le__(self, obj):
    return Fn.LessThanEq(self, obj)

  # arithmetic overrides
  def __mul__(self, other):
    return Fn.Mul(self, other)

  def __pow__(self, other):
    return Fn.Pow(self, other)

  def __rpow__(self, other):
    return Fn.Pow(other, self)

  def __div__(self, other):
    return Fn.Div(self, other)

  def __add__(self, other):
    return Fn.Add(self, other)

  def __sub__(self, other):
    return Fn.Sub(self, other)

  def __or__(self, other):
    return Fn.BitOr(self, other)

  def __and__(self, other):
    return Fn.BitAnd(self, other)

  def __xor__(self, other):
    return Fn.BitXor(self, other)

  def __rmul__(self, other):
    return Fn.Mul(other, self)

  def __rdiv__(self, other):
    return Fn.Div(other, self)

  def __radd__(self, other):
    return Fn.Add(other, self)

  def __rsub__(self, other):
    return Fn.Sub(other, self)

  def __ror__(self, other):
    return Fn.BitOr(other, self)

  def __rand__(self, other):
    return Fn.BitAnd(other, self)

  def __rxor__(self, other):
    return Fn.BitXor(other, self)

  def __rshift__(self, other):
    return Fn.RShift(self, other)

  def __lshift__(self, other):
    return Fn.LShift(self, other)

  def __rrshift__(self, other):
    return Fn.RShift(other, self)

  def __rlshift__(self, other):
    return Fn.LShift(other, self)

  def __neg__(self):
    return self * -1

class _KnownValue(_Symbolic):
  def value(self):
    raise BaseException('not implemented')

class Boolean(_KnownValue):

  @Memoize
  def __new__(typ, b):
    self = _KnownValue.__new__(typ)
    self.name = str(b)
    self.boolean = b
    return self

  def value(self):
    return bool(self.boolean)

  def __str__(self):
    return str(self.boolean)

  def __repr__(self):
    return str(self)

  def __eq__(self, other):
    if isinstance(other, Boolean):
      return bool(self.boolean) == bool(other.boolean)
    elif isinstance(other, _Symbolic):
      return other.__eq__(self)
    else:
      return bool(self.boolean) == other

class Number(_KnownValue):

  IFORMAT = str
  FFORMAT = str

  @Memoize
  def __new__(typ, n):
    n = float(n)
    self = _KnownValue.__new__(typ)
    self.name = str(n)
    self.n = n
    return self

  def value(self):
    return self.n

  def __eq__(self, other):
    if isinstance(other, Number):
      return self.n == other.n
    elif isinstance(other, _Symbolic):
      return other.__eq__(self)
    else:
      return self.n == other

  def __ne__(self, other):
    if isinstance(other, _Symbolic):
      return super(Number, self).__ne__(other)
    else:
      return self.n != other

  def __str__(self):
    if self.n.is_integer():
      return Number.IFORMAT(int(self.n))
    else:
      return Number.FFORMAT(self.n)

  def __repr__(self):
    return str(self)

class WildResults(object):

  def __init__(self):
    self._hash = {}

  def clear(self):
    self._hash.clear()

  def __setitem__(self, idx, val):
    self._hash.__setitem__(idx, val)

  def __contains__(self, idx):
    return idx in self._hash

  def __getitem__(self, idx):
    return self._hash[idx]

  def __getattr__(self, idx):
    return self[idx]

class Wild(_Symbolic):
  '''
  wilds will be equal to anything, and are used for pattern matching
  '''

  @Memoize
  def __new__(typ, name, **kargs):
    self = _Symbolic.__new__(typ)
    self.name = name
    self.kargs = kargs
    return self

  def __str__(self):
    return self.name

  def __repr__(self):
    return str(self)

  def __call__(self, *args):
    return Fn(self, *args)

  def _dump(self):
    return {
        'type': type(self),
        'name': self.name,
        'kargs': self.kargs,
        'id': id(self)
        }

class Symbol(_Symbolic):
  '''
  symbols with the same name and kargs will be equal
  (and in fact are guaranteed to be the same instance)
  '''

  @Memoize
  def __new__(typ, name, **kargs):
    self = Wild.__new__(typ, name)
    self.name = name
    self.kargs = kargs
    self.is_integer = False # set to true to force domain to integers
    self.is_bitvector = 0 # set to the size of the bitvector if it is a bitvector
    self.is_bool = False # set to true if the symbol represents a boolean value
    return self

  def __str__(self):
    return self.name

  def __repr__(self):
    return str(self)

  def __call__(self, *args):
    return Fn(self, *args)

  def _dump(self):
    return {
        'type': type(self),
        'name': self.name,
        'kargs': self.kargs,
        'id': id(self)
        }

class Fn(_Symbolic):

  @Memoize
  def __new__(typ, fn, *args):
    '''
    arguments: Function, *arguments, **kargs
    valid keyword args:
      commutative (default False) - order of operands is unimportant
    '''

    if None in args:
      raise BaseException('NONE IN ARGS %s %s' % (fn, args))

    if not isinstance(fn, _Symbolic):
      fn = symbolic(fn)
      return Fn.__new__(typ, fn, *args)

    for i in args:
      if not isinstance(i, _Symbolic):
        args = list(map(symbolic, args))
        return Fn.__new__(typ, fn, *args)

    self = _Symbolic.__new__(typ)
    kargs = fn.kargs
    self.kargs = fn.kargs

    self.name = fn.name
    self.fn = fn
    self.args = args

    #import simplify
    #rv = simplify.simplify(self)

    return self

  def _dump(self):
    return {
        'id': id(self),
        'name': self.name,
        'fn': self.fn._dump(),
        'kargs': self.kargs,
        'args': list(map(lambda x: x._dump(), self.args)),
        'orig kargs': self.orig_kargs,
        'orig args': list(map(lambda x: x._dump(), self.orig_args))
        }

  def __call__(self, *args):
    return Fn(self, *args)

  def substitute(self, subs):
    args = list(map(lambda x: x.substitute(subs), self.args))
    newfn = self.fn.substitute(subs)
    self = Fn(newfn, *args)

    if self in subs:
      self = subs[self]

    return self

  def __getitem__(self, n):
    if n == 0:
      return self.fn

    return self.args[n - 1]

  def __len__(self):
    return len(self.args) + 1

  def _get_assoc_arguments(self):
    import simplify
    rv = []

    args = list(self.args)
    def _(a, b):
      if (isinstance(a, Fn) and a.fn == self.fn) and not (isinstance(b, Fn) and b.fn == self.fn):
        return -1

      if (isinstance(b, Fn) and b.fn == self.fn) and not (isinstance(a, Fn) and a.fn == self.fn):
        return 1

      return simplify._order(a, b)

    args.sort(_)

    for i in args:
      if isinstance(i, Fn) and i.fn == self.fn:
        for j in i._get_assoc_arguments():
          rv.append(j)
      else:
        rv.append(i)

    return rv

  @staticmethod
  def LessThan(lhs, rhs):
    return Fn(stdops.LessThan, lhs, rhs)

  @staticmethod
  def GreaterThan(lhs, rhs):
    return Fn(stdops.GreaterThan, lhs, rhs)

  @staticmethod
  def LessThanEq(lhs, rhs):
    return Fn(stdops.LessThanEq, lhs, rhs)

  @staticmethod
  def GreaterThanEq(lhs, rhs):
    return Fn(stdops.GreaterThanEq, lhs, rhs)

  @staticmethod
  def Add(lhs, rhs):
    return Fn(stdops.Add, lhs, rhs)

  @staticmethod
  def Sub(lhs, rhs):
    return Fn(stdops.Sub, lhs, rhs)

  @staticmethod
  def Div(lhs, rhs):
    return Fn(stdops.Div, lhs, rhs)

  @staticmethod
  def Mul(lhs, rhs):
    return Fn(stdops.Mul, lhs, rhs)

  @staticmethod
  def Pow(lhs, rhs):
    return Fn(stdops.Pow, lhs, rhs)

  @staticmethod
  def RShift(lhs, rhs):
    return Fn(stdops.RShift, lhs, rhs)

  @staticmethod
  def LShift(lhs, rhs):
    return Fn(stdops.LShift, lhs, rhs)

  @staticmethod
  def BitAnd(lhs, rhs):
    return Fn(stdops.BitAnd, lhs, rhs)

  @staticmethod
  def BitOr(lhs, rhs):
    return Fn(stdops.BitOr, lhs, rhs)

  @staticmethod
  def BitXor(lhs, rhs):
    return Fn(stdops.BitXor, lhs, rhs)

  def __str__(self):
    if isinstance(self.fn, Symbol) and not self.name[0].isalnum() and len(self.args) == 2:
      return '(%s %s %s)' % (self.args[0], self.name, self.args[1])

    return '%s(%s)' % (self.fn, ','.join(map(str, self.args)))

  def __repr__(self):
    return str(self)

def symbols(symstr, **kargs):
  '''
  takes a string of symbols seperated by whitespace
  returns a tuple of symbols
  '''
  syms = symstr.split(' ')
  if len(syms) == 1:
    return Symbol(syms[0], **kargs)

  rv = []
  for i in syms:
    rv.append(Symbol(i, **kargs))

  return tuple(rv)

def wilds(symstr, **kargs):
  '''
  wilds should match anything
  '''
  syms = symstr.split(' ')
  if len(syms) == 1:
    return Wild(syms[0], **kargs)

  rv = []
  for i in syms:
    rv.append(Wild(i, **kargs))

  return tuple(rv)

def wild(name=None, **kargs):
  if name == None:
    name = ''.join(random.choice(string.ascii_lowercase) for x in range(12))
  return Wild(name, **kargs)

def symbolic(obj, **kargs):
  '''
  makes the symbolic version of an object
  '''
  if type(obj) in [type(0), type(0.0), type(0L)]:
    return Number(obj, **kargs)
  elif type(obj) == type('str'):
    return Symbol(obj, **kargs)
  elif type(obj) == type(True):
    return Boolean(obj, **kargs)
  elif isinstance(obj, _Symbolic):
    return obj
  else:
    msg = "Unknown type (%s) %s passed to symbolic" % (type(obj), obj)
    raise BaseException(msg)

import stdops
<commit_msg>added random symbol generation<commit_after>#!/usr/bin/env python
# this only exists because sympy crashes IDAPython
# for general use sympy is much more complete

import traceback
import types
import copy
import operator
import random
import string
from memoize import Memoize

def collect(exp, fn):
  rv = set()

  def _collect(exp):
    if fn(exp):
      rv.add(exp)
    return exp

  exp.walk(collect)
  return rv

class _Symbolic(tuple):

  def match(self, other, valuestore=None):
    '''
    matches against a pattern, use wilds() to generate wilds

    Example:
      a,b = wilds('a b')
      val = WildsResults()

      if exp.match(a(b + 4), val):
        print val.a
        print val.b
    '''
    import match
    return match.match(self, other, valuestore)

  def __hash__(self):
    return hash(self.name)

  def simplify(self):
    import simplify
    return simplify.simplify(self)

  def walk(self, *fns):
    if len(fns) > 1:
      def _(exp):
        for f in fns:
          exp = f(exp)
        return exp
      return self.walk(_)

    exp = self
    fn = fns[0]
    if len(exp) == 1:
      oldexp = exp
      exp = fn(exp)
      while exp != oldexp:
        oldexp = exp
        exp = fn(exp)

    else:
      args = list(map(lambda x: x.walk(fn), exp.args))
      oldexp = self
      exp = fn(fn(exp[0])(*args))

      #while exp != oldexp:
      #  print '%s => %s' % (oldexp, exp)
      #  oldexp = exp
      #  exp = exp.walk(fn)

    return exp

  def _dump(self):
    return {
        'name': self.name,
        'id': id(self)
        }

  def substitute(self, subs):
    '''
    takes a dictionary of substitutions
    returns itself with substitutions made
    '''
    if self in subs:
      self = subs[self]

    return self

  def __eq__(self, other):
    #return type(self) == type(other) and self.name == other.name
    return id(self) == id(other)

  def __ne__(self, other):
    return not self.__eq__(other)

  def __getitem__(self, num):
    if num == 0:
      return self

    raise BaseException("Invalid index")

  def __len__(self):
    return 1

  # comparison operations notice we don't override __eq__
  def __gt__(self, obj):
    return Fn.GreaterThan(self, obj)

  def __ge__(self, obj):
    return Fn.GreaterThanEq(self, obj)

  def __lt__(self, obj):
    return Fn.LessThan(self, obj)

  def __le__(self, obj):
    return Fn.LessThanEq(self, obj)

  # arithmetic overrides
  def __mul__(self, other):
    return Fn.Mul(self, other)

  def __pow__(self, other):
    return Fn.Pow(self, other)

  def __rpow__(self, other):
    return Fn.Pow(other, self)

  def __div__(self, other):
    return Fn.Div(self, other)

  def __add__(self, other):
    return Fn.Add(self, other)

  def __sub__(self, other):
    return Fn.Sub(self, other)

  def __or__(self, other):
    return Fn.BitOr(self, other)

  def __and__(self, other):
    return Fn.BitAnd(self, other)

  def __xor__(self, other):
    return Fn.BitXor(self, other)

  def __rmul__(self, other):
    return Fn.Mul(other, self)

  def __rdiv__(self, other):
    return Fn.Div(other, self)

  def __radd__(self, other):
    return Fn.Add(other, self)

  def __rsub__(self, other):
    return Fn.Sub(other, self)

  def __ror__(self, other):
    return Fn.BitOr(other, self)

  def __rand__(self, other):
    return Fn.BitAnd(other, self)

  def __rxor__(self, other):
    return Fn.BitXor(other, self)

  def __rshift__(self, other):
    return Fn.RShift(self, other)

  def __lshift__(self, other):
    return Fn.LShift(self, other)

  def __rrshift__(self, other):
    return Fn.RShift(other, self)

  def __rlshift__(self, other):
    return Fn.LShift(other, self)

  def __neg__(self):
    return self * -1

class _KnownValue(_Symbolic):
  def value(self):
    raise BaseException('not implemented')

class Boolean(_KnownValue):

  @Memoize
  def __new__(typ, b):
    self = _KnownValue.__new__(typ)
    self.name = str(b)
    self.boolean = b
    return self

  def value(self):
    return bool(self.boolean)

  def __str__(self):
    return str(self.boolean)

  def __repr__(self):
    return str(self)

  def __eq__(self, other):
    if isinstance(other, Boolean):
      return bool(self.boolean) == bool(other.boolean)
    elif isinstance(other, _Symbolic):
      return other.__eq__(self)
    else:
      return bool(self.boolean) == other

class Number(_KnownValue):

  IFORMAT = str
  FFORMAT = str

  @Memoize
  def __new__(typ, n):
    n = float(n)
    self = _KnownValue.__new__(typ)
    self.name = str(n)
    self.n = n
    return self

  def value(self):
    return self.n

  def __eq__(self, other):
    if isinstance(other, Number):
      return self.n == other.n
    elif isinstance(other, _Symbolic):
      return other.__eq__(self)
    else:
      return self.n == other

  def __ne__(self, other):
    if isinstance(other, _Symbolic):
      return super(Number, self).__ne__(other)
    else:
      return self.n != other

  def __str__(self):
    if self.n.is_integer():
      return Number.IFORMAT(int(self.n))
    else:
      return Number.FFORMAT(self.n)

  def __repr__(self):
    return str(self)

class WildResults(object):

  def __init__(self):
    self._hash = {}

  def clear(self):
    self._hash.clear()

  def __setitem__(self, idx, val):
    self._hash.__setitem__(idx, val)

  def __contains__(self, idx):
    return idx in self._hash

  def __getitem__(self, idx):
    return self._hash[idx]

  def __getattr__(self, idx):
    return self[idx]

class Wild(_Symbolic):
  '''
  wilds will be equal to anything, and are used for pattern matching
  '''

  @Memoize
  def __new__(typ, name, **kargs):
    self = _Symbolic.__new__(typ)
    self.name = name
    self.kargs = kargs
    return self

  def __str__(self):
    return self.name

  def __repr__(self):
    return str(self)

  def __call__(self, *args):
    return Fn(self, *args)

  def _dump(self):
    return {
        'type': type(self),
        'name': self.name,
        'kargs': self.kargs,
        'id': id(self)
        }

class Symbol(_Symbolic):
  '''
  symbols with the same name and kargs will be equal
  (and in fact are guaranteed to be the same instance)
  '''

  @Memoize
  def __new__(typ, name, **kargs):
    self = Wild.__new__(typ, name)
    self.name = name
    self.kargs = kargs
    self.is_integer = False # set to true to force domain to integers
    self.is_bitvector = 0 # set to the size of the bitvector if it is a bitvector
    self.is_bool = False # set to true if the symbol represents a boolean value
    return self

  def __str__(self):
    return self.name

  def __repr__(self):
    return str(self)

  def __call__(self, *args):
    return Fn(self, *args)

  def _dump(self):
    return {
        'type': type(self),
        'name': self.name,
        'kargs': self.kargs,
        'id': id(self)
        }

class Fn(_Symbolic):

  @Memoize
  def __new__(typ, fn, *args):
    '''
    arguments: Function, *arguments, **kargs
    valid keyword args:
      commutative (default False) - order of operands is unimportant
    '''

    if None in args:
      raise BaseException('NONE IN ARGS %s %s' % (fn, args))

    if not isinstance(fn, _Symbolic):
      fn = symbolic(fn)
      return Fn.__new__(typ, fn, *args)

    for i in args:
      if not isinstance(i, _Symbolic):
        args = list(map(symbolic, args))
        return Fn.__new__(typ, fn, *args)

    self = _Symbolic.__new__(typ)
    kargs = fn.kargs
    self.kargs = fn.kargs

    self.name = fn.name
    self.fn = fn
    self.args = args

    #import simplify
    #rv = simplify.simplify(self)

    return self

  def _dump(self):
    return {
        'id': id(self),
        'name': self.name,
        'fn': self.fn._dump(),
        'kargs': self.kargs,
        'args': list(map(lambda x: x._dump(), self.args)),
        'orig kargs': self.orig_kargs,
        'orig args': list(map(lambda x: x._dump(), self.orig_args))
        }

  def __call__(self, *args):
    return Fn(self, *args)

  def substitute(self, subs):
    args = list(map(lambda x: x.substitute(subs), self.args))
    newfn = self.fn.substitute(subs)
    self = Fn(newfn, *args)

    if self in subs:
      self = subs[self]

    return self

  def __getitem__(self, n):
    if n == 0:
      return self.fn

    return self.args[n - 1]

  def __len__(self):
    return len(self.args) + 1

  def _get_assoc_arguments(self):
    import simplify
    rv = []

    args = list(self.args)
    def _(a, b):
      if (isinstance(a, Fn) and a.fn == self.fn) and not (isinstance(b, Fn) and b.fn == self.fn):
        return -1

      if (isinstance(b, Fn) and b.fn == self.fn) and not (isinstance(a, Fn) and a.fn == self.fn):
        return 1

      return simplify._order(a, b)

    args.sort(_)

    for i in args:
      if isinstance(i, Fn) and i.fn == self.fn:
        for j in i._get_assoc_arguments():
          rv.append(j)
      else:
        rv.append(i)

    return rv

  @staticmethod
  def LessThan(lhs, rhs):
    return Fn(stdops.LessThan, lhs, rhs)

  @staticmethod
  def GreaterThan(lhs, rhs):
    return Fn(stdops.GreaterThan, lhs, rhs)

  @staticmethod
  def LessThanEq(lhs, rhs):
    return Fn(stdops.LessThanEq, lhs, rhs)

  @staticmethod
  def GreaterThanEq(lhs, rhs):
    return Fn(stdops.GreaterThanEq, lhs, rhs)

  @staticmethod
  def Add(lhs, rhs):
    return Fn(stdops.Add, lhs, rhs)

  @staticmethod
  def Sub(lhs, rhs):
    return Fn(stdops.Sub, lhs, rhs)

  @staticmethod
  def Div(lhs, rhs):
    return Fn(stdops.Div, lhs, rhs)

  @staticmethod
  def Mul(lhs, rhs):
    return Fn(stdops.Mul, lhs, rhs)

  @staticmethod
  def Pow(lhs, rhs):
    return Fn(stdops.Pow, lhs, rhs)

  @staticmethod
  def RShift(lhs, rhs):
    return Fn(stdops.RShift, lhs, rhs)

  @staticmethod
  def LShift(lhs, rhs):
    return Fn(stdops.LShift, lhs, rhs)

  @staticmethod
  def BitAnd(lhs, rhs):
    return Fn(stdops.BitAnd, lhs, rhs)

  @staticmethod
  def BitOr(lhs, rhs):
    return Fn(stdops.BitOr, lhs, rhs)

  @staticmethod
  def BitXor(lhs, rhs):
    return Fn(stdops.BitXor, lhs, rhs)

  def __str__(self):
    if isinstance(self.fn, Symbol) and not self.name[0].isalnum() and len(self.args) == 2:
      return '(%s %s %s)' % (self.args[0], self.name, self.args[1])

    return '%s(%s)' % (self.fn, ','.join(map(str, self.args)))

  def __repr__(self):
    return str(self)

def symbols(symstr=None, **kargs):
  '''
  takes a string of symbols seperated by whitespace
  returns a tuple of symbols
  '''
  if symstr == None:
    syms = [''.join(random.choice(string.ascii_lowercase) for x in range(12))]
  else:
    syms = symstr.split(' ')

  if len(syms) == 1:
    return Symbol(syms[0], **kargs)

  rv = []
  for i in syms:
    rv.append(Symbol(i, **kargs))

  return tuple(rv)

def wilds(symstr, **kargs):
  '''
  wilds should match anything
  '''
  syms = symstr.split(' ')
  if len(syms) == 1:
    return Wild(syms[0], **kargs)

  rv = []
  for i in syms:
    rv.append(Wild(i, **kargs))

  return tuple(rv)

def wild(name=None, **kargs):
  if name == None:
    name = ''.join(random.choice(string.ascii_lowercase) for x in range(12))
  return Wild(name, **kargs)

def symbolic(obj, **kargs):
  '''
  makes the symbolic version of an object
  '''
  if type(obj) in [type(0), type(0.0), type(0L)]:
    return Number(obj, **kargs)
  elif type(obj) == type('str'):
    return Symbol(obj, **kargs)
  elif type(obj) == type(True):
    return Boolean(obj, **kargs)
  elif isinstance(obj, _Symbolic):
    return obj
  else:
    msg = "Unknown type (%s) %s passed to symbolic" % (type(obj), obj)
    raise BaseException(msg)

import stdops
<|endoftext|>


<commit_before>
class SynErr(Exception):

    def __init__(self, **info):
        self.errinfo = info
        Exception.__init__(self, self._getExcMsg() )

    def _getExcMsg(self):
        props = list(self.errinfo.items())
        props.sort()
        displ = ' '.join(['%s=%r' % (p,v) for (p,v) in props ])
        return '%s: %s' % (self.__class__.__name__,displ)

class NoSuchOpt(SynErr):pass
class NoSuchConf(SynErr):pass
class NoSuchForm(SynErr):pass
class NoSuchType(SynErr):pass
class NoSuchProp(SynErr):pass
class NoSuchOper(SynErr):pass
class NoSuchRule(SynErr):pass

class BadTypeValu(SynErr):pass
class DupTypeName(SynErr):pass
class DupPropName(SynErr):pass
class BadPropName(SynErr):pass
class BadMesgVers(SynErr):pass

class SyntaxError(SynErr):pass

class BadUrl(Exception):pass
class BadJson(Exception):pass
class BadMesgResp(Exception):pass
class BadPropValu(SynErr):pass
class BadPySource(Exception):pass

class DupOpt(Exception):pass
class DupUser(Exception):pass
class DupRole(Exception):pass

class NoSuch(Exception):pass
class NoSuchAct(Exception):pass
class NoSuchJob(Exception):pass
class NoSuchDir(Exception):pass
class NoSuchMod(Exception):pass
class NoSuchObj(Exception):pass
class NoSuchFile(Exception):pass
class NoSuchImpl(Exception):pass
class NoSuchIden(Exception):pass
class NoSuchMeth(Exception):pass
class NoSuchFunc(Exception):pass
class NoSuchPeer(Exception):pass
class NoSuchPath(Exception):pass
class NoSuchSess(Exception):pass
class NoSuchUser(Exception):pass
class NoSuchTufo(Exception):pass
class NoSuchRole(Exception):pass
class NoSuchProto(Exception):pass

class NoInitCore(Exception):pass # API disabled because no cortex
class NoCurrSess(Exception):pass # API requires a current session

class SidNotFound(Exception):pass
class PropNotFound(Exception):pass

class HitMaxTime(Exception):pass
class HitMaxRetry(Exception):pass

class MustNotWait(Exception):pass   # blocking function called by no-wait thread

class IsFini(Exception):pass

class JobErr(Exception):
    '''
    Used for remote exception propigation.
    '''
    def __init__(self, job):
        self.job = job

        err = job[1].get('err')
        errmsg = job[1].get('errmsg')
        errfile = job[1].get('errfile')
        errline = job[1].get('errline')

        Exception.__init__(self, '%s: %s (%s:%s)' % (err,errmsg,errfile,errline))

class LinkErr(Exception):

    retry = False
    def __init__(self, link, mesg=''):
        self.link = link
        Exception.__init__(self,'%s %s' % (link[1].get('url'), mesg))

class LinkRefused(LinkErr):
    retry = True

class LinkNotAuth(LinkErr):pass
<commit_msg>whups, forgot the exceptions module<commit_after>
class SynErr(Exception):

    def __init__(self, **info):
        self.errinfo = info
        Exception.__init__(self, self._getExcMsg() )

    def _getExcMsg(self):
        props = list(self.errinfo.items())
        props.sort()
        displ = ' '.join(['%s=%r' % (p,v) for (p,v) in props ])
        return '%s: %s' % (self.__class__.__name__,displ)

class NoSuchOpt(SynErr):pass
class NoSuchConf(SynErr):pass
class NoSuchForm(SynErr):pass
class NoSuchType(SynErr):pass
class NoSuchProp(SynErr):pass
class NoSuchOper(SynErr):pass
class NoSuchRule(SynErr):pass

class BadOperArg(SynErr):pass
class BadTypeValu(SynErr):pass
class DupTypeName(SynErr):pass
class DupPropName(SynErr):pass
class BadPropName(SynErr):pass
class BadMesgVers(SynErr):pass
class BadInfoValu(SynErr):pass
class BadStorValu(SynErr):pass

class WebAppErr(SynErr):pass

class SyntaxError(SynErr):pass

class BadUrl(Exception):pass
class BadJson(Exception):pass
class BadMesgResp(Exception):pass
class BadPropValu(SynErr):pass
class BadPySource(Exception):pass

class DupOpt(Exception):pass
class DupUser(Exception):pass
class DupRole(Exception):pass

class NoSuch(Exception):pass
class NoSuchAct(Exception):pass
class NoSuchJob(Exception):pass
class NoSuchDir(Exception):pass
class NoSuchMod(Exception):pass
class NoSuchObj(Exception):pass
class NoSuchFile(Exception):pass
class NoSuchImpl(Exception):pass
class NoSuchIden(Exception):pass
class NoSuchMeth(Exception):pass
class NoSuchFunc(Exception):pass
class NoSuchPeer(Exception):pass
class NoSuchPath(Exception):pass
class NoSuchSess(Exception):pass
class NoSuchUser(Exception):pass
class NoSuchTufo(Exception):pass
class NoSuchRole(Exception):pass
class NoSuchProto(Exception):pass

class NoInitCore(Exception):pass # API disabled because no cortex
class NoCurrSess(Exception):pass # API requires a current session

class SidNotFound(Exception):pass
class PropNotFound(Exception):pass

class HitMaxTime(Exception):pass
class HitMaxLimit(Exception):pass
class HitMaxRetry(Exception):pass

class MustNotWait(Exception):pass   # blocking function called by no-wait thread

class IsFini(Exception):pass

class JobErr(Exception):
    '''
    Used for remote exception propigation.
    '''
    def __init__(self, job):
        self.job = job

        err = job[1].get('err')
        errmsg = job[1].get('errmsg')
        errfile = job[1].get('errfile')
        errline = job[1].get('errline')

        Exception.__init__(self, '%s: %s (%s:%s)' % (err,errmsg,errfile,errline))

class LinkErr(Exception):

    retry = False
    def __init__(self, link, mesg=''):
        self.link = link
        Exception.__init__(self,'%s %s' % (link[1].get('url'), mesg))

class LinkRefused(LinkErr):
    retry = True

class LinkNotAuth(LinkErr):pass
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright 2013 Russell Heilling
# pylint: disable=C0103,R0903
"""Sync items on toread list from calibre database into local folder.

Processes a toread file and exports the files for the first listed
titles listed into a local folder.

The toread file shold be in the format:

{comicvine_id} {title}

The content of the title field are not used.  This format is
compatible with todo.txt.
"""
from difflib import SequenceMatcher
import logging
from math import ceil
import os
import re

from collections import OrderedDict

import args
from calibredb import CalibreDB, set_log_level
import logs

args.add_argument('--count', '-c', help='Number of issues to sync',
                  type=int, default='50')
args.add_argument('--toread', '-t', help='File containing issues to read',
                  type=bytes, default=None, required=True)
args.add_argument('--syncdir', '-d', help='Directory to sync issues to',
                  type=bytes, default=None, required=True)

ARGS = args.ARGS

class FormatChangeError(Exception):
  'Exception raised when attempt made to change format during rename'

class ReindexError(Exception):
  'Exception raised when reindexing is not possible'

class ToRead(OrderedDict):
  'Read matching lines from the toread file into an ordered dict'
  title_pattern = re.compile(r'^(\d+)\s+(.*)$')

  def __init__(self, toread_file, validator=None):
    super(ToRead, self).__init__()
    with open(toread_file, 'r') as toread:
      for title in toread:
        title_match = self.title_pattern.match(title)
        if title_match:
          if validator:
            try:
              validator(int(title_match.group(1)))
            except TypeError:
              # Raised when validator is not callable
              raise
            except Exception as err:
              logging.warn('IssueID not valid in line: %s', title)
              continue
          self[title_match.group(1)] = title_match.group(2)
          logging.debug('Found title %s(%s)', title_match.group(2),
                        title_match.group(1))
        else:
          logging.info('Unmatching toread line: %s', title)

class ExportDirectory(dict):
  'Files present in the export directory'
  title_pattern = re.compile(r'\((\d+)\).(cb[rz])')

  def __init__(self, directory):
    super(ExportDirectory, self).__init__()
    self.directory = directory
    self.format = {}
    self.scan()

  def __delitem__(self, key):
    if os.path.exists(os.path.join(self.directory, self[key])):
      try:
        os.remove(os.path.join(self.directory, self[key]))
      except OSError as err:
        logging.error('Error removing file %s: %s', self[key], err)
    super(ExportDirectory, self).__delitem__(key)
    del self.format[key]

  def __setitem__(self, key, value):
    title_match = self.title_pattern.search(value)
    if not title_match:
      raise ValueError('Cannot find ID and extension for file %s' % value)
    calibre_id = title_match.group(1)
    if key != calibre_id:
      raise KeyError(
        'Calibre ID in file(%s) does not match specified key(%s)' % (
          calibre_id, key))

    # Setting a key that exists should rename the file
    if key in self and os.path.exists(os.path.join(self.directory,
                                                   self[key])):
      if self[calibre_id] == value:
        logging.debug('Skipping rename, destination same as source: %s',
                      self[calibre_id])
        return
      if not value.endswith(self.format[calibre_id]):
        raise FormatChangeError(
          'Attempt to change format during rename (%s does not have '
          'format %s).', value, self.format[calibre_id])
      logging.debug('Renaming %s to %s', self[key], value)
      try:
        os.rename(os.path.join(self.directory, self[key]),
                  os.path.join(self.directory, value))
      except OSError as err:
        raise OSError ('Error renaming file %s -> %s: %s',
                       self[key], value, err)
    elif not os.path.exists(os.path.join(self.directory, value)):
      raise ValueError('File %s does not exist' % value)

    super(ExportDirectory, self).__setitem__(key, value)
    self.format[key] = title_match.group(2)

  def scan(self):
    'Scan the export directory for files'
    for key in self:
      if not os.path.exists(os.path.join(self.directory, self[key])):
        del self[key]
    for entry in os.listdir(self.directory):
      id_match = self.title_pattern.search(entry)
      if id_match:
        calibre_id = id_match.group(1)
        if calibre_id not in self:
          logging.debug('Found file %s(%s)', entry, id_match.group(1))
          self[calibre_id] = entry

  def keep_files(self, keep_ids):
    'Remove any files that are not in the list of ids to keep.'
    for calibre_id, filename in self.items():
      if calibre_id not in keep_ids:
        logging.info('Removing file %s', filename)
        del self[calibre_id]

  def find_missing(self, wanted):
    'Return list of ids from wanted list for which there are no files present.'
    return list(set(wanted) - set(self.keys()))

def file_index(filename):
  'Find file index in filename if present.'
  index_match = re.match(r'(\d{4}(?:\.\d{2,3})?) ', filename)
  if index_match:
    index = index_match.group(1)
    return float(index)

def ordered_files(syncdir, toread):
  'Find the largest set of files in the correct order.'
  # Create a dict of files in syncdir with a valid index
  # In case of index clashes, the first candidate wins
  valid_files = {}
  for calibreid, filename in syncdir.items():
    index = file_index(filename)
    if index and index not in valid_files:
      valid_files[index] = calibreid
  # Create lists to compare to find the largest ordered subset
  synclist = [valid_files[index] for index in sorted(valid_files.keys())]
  toreadlist = toread.keys()[:ARGS.count]
  logging.debug('Comparing %r and %r', synclist, toreadlist)
  matcher = SequenceMatcher(None, toreadlist, synclist)
  ordered_ids = []
  for i, j, count in matcher.get_matching_blocks():
    ordered_ids.extend(toreadlist[i:i+count])
  logging.debug('Longest sorted subset: %r', ([
      syncdir[title] for title in ordered_ids],))
  return ordered_ids

def new_indexes(start, finish, titles):
  'Find new indexes for titles that fit between start and finish.'
  logging.debug('Inserting %d titles between %s and %s',
                len(titles), start, finish)
  indexes = []
  if not finish:
    # Simplest case - just append titles with increasing integer indexes
    # Space is not an issue so leave gap for future reshuffles
    logging.debug('Appending %d titles to end of list', len(titles))
    finish = ceil(start) + 10 * len(titles)
  # Fit the titles between the start and finish with even spacing.
  # Round issues that are near an integer to the whole number to try
  # and avoid everything going fractional...
  interval = (finish - start) / (len(titles)+1)
  if round(finish,3) <= round(start,3) or interval < 1e-3:
    raise ReindexError('Unable in insert %d issues between %f and %f (%f)' %
                       (len(titles), start, finish, interval))
  logging.debug('Stepping from %r to %r with interval %r', start,
                finish, interval)
  index = start
  for title in titles:
    index += interval
    int_next = int(index+interval)
    if index+interval > int_next and int_next > int(index):
      # We are just about to cross an integer boundary.  Put the
      # issue on the boundary rather than just past it.
      indexes.append('%08.3f' % float(int_next))
    else:
      indexes.append('%08.3f' % index)
  return zip(indexes, titles)

def rename_files(syncdir, toread):
  'Rename files so that the filenames sort in todolist order.'
  # Get ordered files
  good_files = ordered_files(syncdir, toread)
  good_ratio = float(len(good_files)) / len(syncdir)
  level = logging.INFO
  if good_ratio < 0.5:
    level = logging.WARN
  logging.debug('%d good %d total', len(good_files), len(syncdir))
  logging.log(
    level, 'Renaming %.2f%% of files (%d/%d)',
    100*(1-good_ratio), len(syncdir) - len(good_files), len(syncdir))
  # iterate through todo items
  last_index = 0
  rename_queue = []
  reindex_queue = []
  for title in toread.keys()[:ARGS.count]:
    #  if item in good_files reindex any pending files
    if title in good_files:
      current_index = file_index(syncdir[title])
      logging.debug('File has suitable index, ignoring %s (i:%d)',
                    syncdir[title], current_index)
      if reindex_queue:
        try:
          reindex_entries = new_indexes(last_index, current_index,
                                        reindex_queue)
        except ReindexError as err:
          logging.error('Error reindexing files: %s', err.message)
          good_files = []
        else:
          logging.debug('New indices: %r', [
              (index, toread[title]) for index, title in reindex_entries])
          rename_queue.extend(reindex_entries)
          reindex_queue = []
      last_index = current_index
      continue
    else:
      logging.debug('Adding %s to reindex queue', syncdir[title])
      reindex_queue.append(title)
  # Append any remaning files in the reindex queue to the rename queue
  # These are going after the issue so we can let new_indexes know
  # just to assign them integers
  if reindex_queue:
    rename_queue.extend(new_indexes(last_index, None, reindex_queue))

  for index, title in rename_queue:
    newname = '%s %s (%s).%s' % (index, toread[title], title,
                                 syncdir.format[title])
    # Remove dangerous characters from title
    newname = re.sub(r'[:\'/"!]', r'_', newname)
    if syncdir[title] == newname:
      continue
    logging.info('Renaming %s to %s', syncdir[title], newname)
    try:
      syncdir[title] = newname
    except OSError:
      logging.warn(OSError)

def main():
  'Read the toread list'
  calibredb = CalibreDB()
  toread = ToRead(ARGS.toread, validator=calibredb.issue)
  syncdir = ExportDirectory(ARGS.syncdir)

  # Grab the ids of the first count entries
  wanted = toread.keys()[:ARGS.count]

  # Remove any files not in the list
  syncdir.keep_files(wanted)

  # Export any files not already present
  calibredb.export_files(wanted, syncdir)

  # Rename files so they sort in reading list order
  rename_files(syncdir, toread)

if __name__ == '__main__':
  args.parse_args()
  logs.set_logging()
  main()
<commit_msg>Fix index rounding.<commit_after>#!/usr/bin/python
# Copyright 2013 Russell Heilling
# pylint: disable=C0103,R0903
"""Sync items on toread list from calibre database into local folder.

Processes a toread file and exports the files for the first listed
titles listed into a local folder.

The toread file shold be in the format:

{comicvine_id} {title}

The content of the title field are not used.  This format is
compatible with todo.txt.
"""
from difflib import SequenceMatcher
import logging
from math import ceil, floor
import os
import re

from collections import OrderedDict

import args
from calibredb import CalibreDB, set_log_level
import logs

args.add_argument('--count', '-c', help='Number of issues to sync',
                  type=int, default='50')
args.add_argument('--toread', '-t', help='File containing issues to read',
                  type=bytes, default=None, required=True)
args.add_argument('--syncdir', '-d', help='Directory to sync issues to',
                  type=bytes, default=None, required=True)

ARGS = args.ARGS

class FormatChangeError(Exception):
  'Exception raised when attempt made to change format during rename'

class ReindexError(Exception):
  'Exception raised when reindexing is not possible'

class ToRead(OrderedDict):
  'Read matching lines from the toread file into an ordered dict'
  title_pattern = re.compile(r'^(\d+)\s+(.*)$')

  def __init__(self, toread_file, validator=None):
    super(ToRead, self).__init__()
    with open(toread_file, 'r') as toread:
      for title in toread:
        title_match = self.title_pattern.match(title)
        if title_match:
          if validator:
            try:
              validator(int(title_match.group(1)))
            except TypeError:
              # Raised when validator is not callable
              raise
            except Exception as err:
              logging.warn('IssueID not valid in line: %s', title)
              continue
          self[title_match.group(1)] = title_match.group(2)
          logging.debug('Found title %s(%s)', title_match.group(2),
                        title_match.group(1))
        else:
          logging.info('Unmatching toread line: %s', title)

class ExportDirectory(dict):
  'Files present in the export directory'
  title_pattern = re.compile(r'\((\d+)\).(cb[rz])')

  def __init__(self, directory):
    super(ExportDirectory, self).__init__()
    self.directory = directory
    self.format = {}
    self.scan()

  def __delitem__(self, key):
    if os.path.exists(os.path.join(self.directory, self[key])):
      try:
        os.remove(os.path.join(self.directory, self[key]))
      except OSError as err:
        logging.error('Error removing file %s: %s', self[key], err)
    super(ExportDirectory, self).__delitem__(key)
    del self.format[key]

  def __setitem__(self, key, value):
    title_match = self.title_pattern.search(value)
    if not title_match:
      raise ValueError('Cannot find ID and extension for file %s' % value)
    calibre_id = title_match.group(1)
    if key != calibre_id:
      raise KeyError(
        'Calibre ID in file(%s) does not match specified key(%s)' % (
          calibre_id, key))

    # Setting a key that exists should rename the file
    if key in self and os.path.exists(os.path.join(self.directory,
                                                   self[key])):
      if self[calibre_id] == value:
        logging.debug('Skipping rename, destination same as source: %s',
                      self[calibre_id])
        return
      if not value.endswith(self.format[calibre_id]):
        raise FormatChangeError(
          'Attempt to change format during rename (%s does not have '
          'format %s).', value, self.format[calibre_id])
      logging.debug('Renaming %s to %s', self[key], value)
      try:
        os.rename(os.path.join(self.directory, self[key]),
                  os.path.join(self.directory, value))
      except OSError as err:
        raise OSError ('Error renaming file %s -> %s: %s',
                       self[key], value, err)
    elif not os.path.exists(os.path.join(self.directory, value)):
      raise ValueError('File %s does not exist' % value)

    super(ExportDirectory, self).__setitem__(key, value)
    self.format[key] = title_match.group(2)

  def scan(self):
    'Scan the export directory for files'
    for key in self:
      if not os.path.exists(os.path.join(self.directory, self[key])):
        del self[key]
    for entry in os.listdir(self.directory):
      id_match = self.title_pattern.search(entry)
      if id_match:
        calibre_id = id_match.group(1)
        if calibre_id not in self:
          logging.debug('Found file %s(%s)', entry, id_match.group(1))
          self[calibre_id] = entry

  def keep_files(self, keep_ids):
    'Remove any files that are not in the list of ids to keep.'
    for calibre_id, filename in self.items():
      if calibre_id not in keep_ids:
        logging.info('Removing file %s', filename)
        del self[calibre_id]

  def find_missing(self, wanted):
    'Return list of ids from wanted list for which there are no files present.'
    return list(set(wanted) - set(self.keys()))

def file_index(filename):
  'Find file index in filename if present.'
  index_match = re.match(r'(\d{4}(?:\.\d{2,3})?) ', filename)
  if index_match:
    index = index_match.group(1)
    return float(index)

def ordered_files(syncdir, toread):
  'Find the largest set of files in the correct order.'
  # Create a dict of files in syncdir with a valid index
  # In case of index clashes, the first candidate wins
  valid_files = {}
  for calibreid, filename in syncdir.items():
    index = file_index(filename)
    if index and index not in valid_files:
      valid_files[index] = calibreid
  # Create lists to compare to find the largest common sequence
  synclist = [valid_files[index] for index in sorted(valid_files.keys())]
  toreadlist = toread.keys()[:ARGS.count]
  logging.debug('Comparing %r and %r', synclist, toreadlist)
  # Use diffutils.SequenceMatcher to do the heavy lifting
  matcher = SequenceMatcher(None, toreadlist, synclist)
  ordered_ids = []
  for i, j, count in matcher.get_matching_blocks():
    ordered_ids.extend(toreadlist[i:i+count])
  logging.debug('Longest sorted subset: %r', ([
      syncdir[title] for title in ordered_ids],))
  return ordered_ids

def new_indexes(start, finish, titles):
  'Find new indexes for titles that fit between start and finish.'
  logging.debug('Inserting %d titles between %s and %s',
                len(titles), start, finish)
  indexes = []
  if not finish:
    # Simplest case - just append titles with increasing integer indexes
    # Space is not an issue so leave gap for future reshuffles
    logging.debug('Appending %d titles to end of list', len(titles))
    finish = ceil(start) + 10 * len(titles)
  # Fit the titles between the start and finish with even spacing.
  # Round issues that are near an integer to the whole number to try
  # and avoid everything going fractional...
  interval = (finish - start) / (len(titles)+1)
  if round(finish,3) <= round(start,3) or interval < 1e-2:
    raise ReindexError('Unable in insert %d issues between %f and %f (%f)' %
                       (len(titles), start, finish, interval))
  logging.debug('Stepping from %r to %r with interval %r', start,
                finish, interval)
  index = start
  for title in titles:
    index += interval
    if index - int(index) < interval:
      # We have just crossed an integer boundary.  Put the
      # issue on the boundary rather than just past it.
      indexes.append('%08.3f' % floor(index))
    else:
      indexes.append('%08.3f' % index)
  return zip(indexes, titles)

def process_rename_queue(syncdir, toread, rename_queue):
  # Now handle the actual renaming
  for index, title in rename_queue:
    newname = '%s %s (%s).%s' % (index, toread[title], title,
                                 syncdir.format[title])
    # Remove dangerous characters from title
    newname = re.sub(r'[:\'/"!]', r'_', newname)
    if syncdir[title] == newname:
      continue
    logging.info('Renaming %s to %s', syncdir[title], newname)
    try:
      syncdir[title] = newname
    except OSError:
      logging.warn(OSError)

def rename_files(syncdir, toread):
  'Rename files so that the filenames sort in todolist order.'
  # Get ordered files
  good_files = ordered_files(syncdir, toread)
  good_ratio = float(len(good_files)) / len(syncdir)
  level = logging.INFO
  if good_ratio < 0.5:
    level = logging.WARN
  logging.debug('%d good %d total', len(good_files), len(syncdir))
  logging.log(
    level, 'Renaming %.2f%% of files (%d/%d)',
    100*(1-good_ratio), len(syncdir) - len(good_files), len(syncdir))
  # iterate through todo items
  last_index = 0
  rename_queue = []
  reindex_queue = []
  for title in toread.keys()[:ARGS.count]:
    #  if item in good_files reindex any pending files
    if title in good_files:
      current_index = file_index(syncdir[title])
      logging.debug('File has suitable index, ignoring %s (i:%d)',
                    syncdir[title], current_index)
      if reindex_queue:
        try:
          reindex_entries = new_indexes(last_index, current_index,
                                        reindex_queue)
        except ReindexError as err:
          # If there is an error indexing the files, abort the attempt
          # to keep the same indexes and just renumber the rest of the
          # files.
          logging.error('Error reindexing files: %s', err.message)
          good_files = []
        else:
          logging.debug('New indices: %r', [
              (index, toread[title]) for index, title in reindex_entries])
          rename_queue.extend(reindex_entries)
          reindex_queue = []
      last_index = current_index
      continue
    else:
      logging.debug('Adding %s to reindex queue', syncdir[title])
      reindex_queue.append(title)
  # Append any remaning files in the reindex queue to the rename queue
  # These are going on the end so we can let new_indexes have as much
  # room as it wants
  if reindex_queue:
    rename_queue.extend(new_indexes(last_index, None, reindex_queue))

  process_rename_queue(syncdir, toread, rename_queue)

def main():
  'Read the toread list'
  calibredb = CalibreDB()
  toread = ToRead(ARGS.toread, validator=calibredb.issue)
  syncdir = ExportDirectory(ARGS.syncdir)

  # Grab the ids of the first count entries
  wanted = toread.keys()[:ARGS.count]

  # Remove any files not in the list
  syncdir.keep_files(wanted)

  # Export any files not already present
  calibredb.export_files(wanted, syncdir)

  # Rename files so they sort in reading list order
  rename_files(syncdir, toread)

if __name__ == '__main__':
  args.parse_args()
  logs.set_logging()
  main()
<|endoftext|>


<commit_before>"""See more details at:

https://github.com/freelawproject/courtlistener/issues/311

The process will be to iterate over the oral arguments that were part of the
Brad Heath import and to use the download URL as our anchor when fixing
everything.

1. Import a fresh copy of the CSV and store it in a data structure that makes
   it possible to look up items by download URL.

1. For each item in the brad heath collection:
     - use the download URL to look up the correct meta data
     - create a new audio item using the original mp3 on disk, the local mp3 on
       disk and the correct meta data.
     - create a new docket to go with that item or update the original docket.
     - update the ID3 data for the mp3.
     - delete:
        - the old Audio item
        - the original files associated with it (both of them).
        - the docket associated with it.

1. Wait for or manually send a commit to Solr.
"""
import os
import sys

execfile('/etc/courtlistener')
sys.path.append(INSTALL_ROOT)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "alert.settings")

from django.core.files.base import ContentFile
import eyed3
from alert.audio.models import Audio
from alert.corpus_importer.brad_heath_oral_arguments.import_oa_from_csv \
    import make_line_to_dict
from alert.lib.string_utils import trunc
from alert.scrapers.tasks import set_mp3_meta_data
from alert.search.models import Court
from juriscraper.AbstractSite import logger

if __name__ == '__main__':
    logger.info("Updating all items part of Brad Heath collection")

    csv = {}
    with open('arguments-from-brad-heath-untouched.csv', 'r') as f:
        # Import a fresh copy of the CSV and store it so we can look things up
        # by URL.
        next(f)
        for row in f:
            item_dict = make_line_to_dict(row)
            csv[item_dict['url']] = item_dict

    for af in Audio.objects.filter(source='H').order_by('time_retrieved'):
        # For every item in the BH archive...
        logger.info("Updating item %s with URL: %s" %
                    (af.pk, af.download_url))

        # sha1_hash --> OK, unchanged.
        # download_url --> OK, unchanged.

        item = csv[af.download_url]
        docket = af.docket

        af.case_name = item['case_name']
        docket.case_name = item['case_name']
        af.date_argued = item['date_argued']

        if item['judges']:
            af.judges = item['judges']
        if item['docket_number']:
            af.docket_number = item['docket_number']

        court = Court.objects.get(pk=item['court_code'])
        docket.court = court

        # Fix the files. First save the location of the old files.
        original_local_path = af.local_path_original_file.path
        original_mp3_path = af.local_path_mp3.path

        # Create a new file with the contents of the old and a corrected
        # name. This is only in memory for the moment.
        cf = ContentFile(af.local_path_original_file.read())
        extension = '.' + af.local_path_original_file.path.rsplit('.', 1)[1]
        file_name = trunc(item['case_name'].lower(), 75) + extension
        af.local_path_original_file.save(file_name, cf, save=False)

        # Create a new mp3 file with the new contents
        cf = ContentFile(af.local_path_mp3.read())
        file_name = trunc(af.case_name.lower(), 72) + '_cl.mp3'
        af.local_path_mp3.save(file_name, cf, save=False)

        # Save things so they can be referenced in a sec.
        docket.save()
        af.save(index=False)

        # Update the ID3 information and duration data.
        set_mp3_meta_data(af, original_mp3_path)
        af.duration = eyed3.load(original_mp3_path).info.time_secs

        docket.save()
        af.save(index=True)

        os.remove(original_local_path)
        os.remove(original_mp3_path)
<commit_msg>Creates a cleanup script for oa issue #311.<commit_after>"""See more details at:

https://github.com/freelawproject/courtlistener/issues/311

The process will be to iterate over the oral arguments that were part of the
Brad Heath import and to use the download URL as our anchor when fixing
everything.

1. Import a fresh copy of the CSV and store it in a data structure that makes
   it possible to look up items by download URL.

1. For each item in the brad heath collection:
     - use the download URL to look up the correct meta data
     - create a new audio item using the original mp3 on disk, the local mp3 on
       disk and the correct meta data.
     - create a new docket to go with that item or update the original docket.
     - update the ID3 data for the mp3.
     - delete:
        - the old Audio item
        - the original files associated with it (both of them).
        - the docket associated with it.

1. Wait for or manually send a commit to Solr.
"""
import os
import sys

execfile('/etc/courtlistener')
sys.path.append(INSTALL_ROOT)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "alert.settings")

from django.core.files.base import ContentFile
from alert.audio.models import Audio
from alert.corpus_importer.brad_heath_oral_arguments.import_oa_from_csv \
    import make_line_to_dict
from alert.lib.string_utils import trunc
from alert.scrapers.tasks import set_mp3_meta_data
from alert.search.models import Court
from juriscraper.AbstractSite import logger

if __name__ == '__main__':
    logger.info("Updating all items part of Brad Heath collection")

    csv = {}
    with open('arguments-from-brad-heath-untouched.csv', 'r') as f:
        # Import a fresh copy of the CSV and store it so we can look things up
        # by URL.
        next(f)
        for row in f:
            item_dict = make_line_to_dict(row)
            csv[item_dict['url']] = item_dict

    for af in Audio.objects.filter(source='H').order_by('time_retrieved'):
        # For every item in the BH archive...
        logger.info("Updating item %s with URL: %s" %
                    (af.pk, af.download_url))

        # sha1_hash --> OK, unchanged.
        # download_url --> OK, unchanged.

        item = csv[af.download_url]
        docket = af.docket

        af.case_name = item['case_name']
        docket.case_name = item['case_name']
        af.date_argued = item['date_argued']

        if item['judges']:
            af.judges = item['judges']
        if item['docket_number']:
            af.docket_number = item['docket_number']

        court = Court.objects.get(pk=item['court_code'])
        docket.court = court

        # Fix the files. First save the location of the old files.
        original_local_path = af.local_path_original_file.path
        original_mp3_path = af.local_path_mp3.path

        # Create a new file with the contents of the old and a corrected
        # name. This is only in memory for the moment.
        cf = ContentFile(af.local_path_original_file.read())
        extension = '.' + af.local_path_original_file.path.rsplit('.', 1)[1]
        file_name = trunc(item['case_name'].lower(), 75) + extension
        af.local_path_original_file.save(file_name, cf, save=False)

        # Create a new mp3 file with the new contents
        cf = ContentFile(af.local_path_mp3.read())
        file_name = trunc(af.case_name.lower(), 72) + '_cl.mp3'
        af.local_path_mp3.save(file_name, cf, save=False)

        # Save things so they can be referenced in a sec.
        docket.save()
        af.save(index=False)

        # Update the ID3 information and duration data.
        set_mp3_meta_data(af, af.local_path_original_file.path)

        docket.save()
        af.save(index=True)

        os.remove(original_local_path)
        os.remove(original_mp3_path)
<|endoftext|>


<commit_before>#
# Copyright (c) 2010 Mandriva
#
# This file is part of python-systemd.
#
# python-systemd is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# python-systemd is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import dbus

from systemd.property import Property

class Job(object):
    """Abstraction class to org.freedesktop.systemd1.Job interface"""
    def __init__(self, job_path):
        self.__bus = dbus.SystemBus()
        self.__proxy = self.__bus.get_object(
            'org.freedesktop.systemd1',
            job_path,
        )
        self.__interface = dbus.Interface(
            self.__proxy,
            'org.freedesktop.systemd1.Job',
        )
        #self.__properties()

    def __properties(self):
        #TODO: Fix it, way Job interface not have properties
        interface = dbus.Interface(
            self.__proxy,
            'org.freedesktop.DBus.Properties')
        properties = interface.GetAll(self.__interface.dbus_interface)
        attr_property =  Property()
        for key, value in properties.items():
            setattr(attr_property, key, value)
        setattr(self, 'properties', attr_property)

    def cancel(self):
        self.__interface.Cancel()

class JobInfo(object):
    def __init__(self):
        self.id = None
        self.name = None
        self.type = None
        self.state = None
        self.job_path = None
        self.unit_path = None<commit_msg>Added exceptions and enable Job properties<commit_after>#
# Copyright (c) 2010 Mandriva
#
# This file is part of python-systemd.
#
# python-systemd is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# python-systemd is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import dbus

from systemd.property import Property
from systemd.exceptions import SystemdError

class Job(object):
    """Abstraction class to org.freedesktop.systemd1.Job interface"""
    def __init__(self, job_path):
        self.__bus = dbus.SystemBus()
        self.__proxy = self.__bus.get_object(
            'org.freedesktop.systemd1',
            job_path,
        )
        self.__interface = dbus.Interface(
            self.__proxy,
            'org.freedesktop.systemd1.Job',
        )
        self.__properties()

    def __properties(self):
        interface = dbus.Interface(
            self.__proxy,
            'org.freedesktop.DBus.Properties')
        properties = interface.GetAll(self.__interface.dbus_interface)
        attr_property =  Property()
        for key, value in properties.items():
            setattr(attr_property, key, value)
        setattr(self, 'properties', attr_property)

    def cancel(self):
        try:
            self.__interface.Cancel()
        except dbus.exceptions.DBusException, error:
            raise SystemdError(error)

class JobInfo(object):
    def __init__(self):
        self.id = None
        self.name = None
        self.type = None
        self.state = None
        self.job_path = None
        self.unit_path = None<|endoftext|>


<commit_before>import cherrypy
import cgi
from configReader import ConfigReader
import os.path
import tempfile

configReader = ConfigReader(name="serverConfig.txt")
keys = configReader.getKeys()
location = keys['location']
class Main():
    @cherrypy.expose
    def index(self):
        raise cherrypy.HTTPError(403)
    @cherrypy.expose
    def upload(self,**kwargs):
            cherrypy.request.body.process()
            parts = cherrypy.request.params['file']
            outFile = open(os.path.join(location,cherrypy.request.params['name']),'w')
            for part in parts:
                outFile.write(part.fullvalue())
            outFile.close()
            print 'written!'
cherrypy.server.socket_host = '0.0.0.0'
cherrypy.quickstart(Main())
<commit_msg>Add support for uwsgi<commit_after>import cherrypy
import cgi
from configReader import ConfigReader
import os.path
import tempfile

configReader = ConfigReader(name="serverConfig.txt")
keys = configReader.getKeys()
location = keys['location']
class Main():
    @cherrypy.expose
    def index(self):
        raise cherrypy.HTTPError(403)
    @cherrypy.expose
    def upload(self,**kwargs):
            cherrypy.request.body.process()
            parts = cherrypy.request.params['file']
            outFile = open(os.path.join(location,cherrypy.request.params['name']),'w')
            if parts != None:
                for part in parts:
                    outFile.write(part.fullvalue())
                outFile.close()
                print 'written!'
            else:
                raise ValueError

application = cherrypy.tree.mount(Main(),'/')
if __name__=='__main__':
    cherrypy.server.socket_host='0.0.0.0'
    cherrypy.engine.start()
    cherrypy.engine.block()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Utility to mark the HEAD with a branch and timestamped tag, and the upstream
commit (that we're rebased on top of) with another timestamped tag.

This allows you to

1) easily find an older set of rebased patches with `git tag`
2) see in tig/gitk which commits you've previously rebased onto.  This is
   useful for seeing which new commits you might need to review.

Sample usage:
git pull --rebase
tagmyrebase.py --branch-head good --tag-head 'good-{YMDN}' --tag-upstream 'U-{YMDN}'

All three arguments are optional.

For any of --tag-head, --branch-head, and --tag-upstream, you can use {YMDHMS}
to insert the current time, or {YMDN} to insert the current date with a
counter that avoids collision with existing tags.
"""

__version__ = '0.2'

import re
import sys
import datetime
import subprocess
import argparse
import pprint

def call(cmd):
	return subprocess.check_call(cmd)

def get_re_for_format_string(format_string):
	"""
	Returns a regular expression for a format string, so that we can check
	if a tag already exists.
	"""
	YMDHMS_RE = r'\d\d\d\d-[01]\d-[0-3]\d_[0-2]\d-[0-5]\d-[0-6]\d'
	YMDN_RE = r'\d\d\d\d-[01]\d-[0-3]\d\.\d+'
	return re.compile(
		r'\A' +
		format_string
			.replace('{YMDHMS}', YMDHMS_RE)
			.replace('{YMDN}', YMDN_RE) +
		r'\Z')

def get_tags_on_commit(commit):
	stdout = subprocess.check_output(["git", "tag", "-l", "--points-at", commit])
	tags = stdout.replace("\r", "").strip("\n").split("\n")
	return tags

def get_all_tags():
	stdout = subprocess.check_output(["git", "tag", "-l"])
	tags = stdout.replace("\r", "").strip("\n").split("\n")
	return tags

def get_expanded_name(format_string, t):
	ymdn = None
	if '{YMDN}' in format_string:
		all_tags = set(get_all_tags())
		ymd = t.strftime('%Y-%m-%d')
		for n in xrange(1, 100000):
			proposed_ymdn = ymd + '.' + str(n)
			proposed_tag = get_expanded_name(
				format_string.format(
					YMDN=proposed_ymdn,
					YMDHMS='{YMDHMS}'
				), t)
			if not proposed_tag in all_tags:
				ymdn = proposed_ymdn
				break
		else:
			raise RuntimeError("100,000 tags in one day is too many tags")

	return format_string.format(
		YMDN=ymdn,
		YMDHMS=t.strftime('%Y-%m-%d_%H-%M-%S')
	)

def get_tag_message(upstream_commit):
	# Annotated tag already has tagger name, e-mail, and date
	# TODO XXX: what if this date doesn't match the date that Python gets?
	# Maybe set GIT_COMMITTER_DATE for git tag
	return "Onto: %s\n" % (upstream_commit,)

def get_reflog_entries(branch_name):
	for line in open(".git/logs/" + branch_name, "rb"):
		before_email, after_email = line.split(">", 1)
		before_email += ">"
		old, new, email = before_email.split(" ", 2)
		_, date, tz = after_email.split("\t", 1)[0].split(" ", 2)
		message = after_email.split("\t", 1)[1]
		assert _ == "", repr(_)
		yield dict(old=old, new=new, email=email, date=date, tz=tz, message=message)

def get_upstream_commit():
	"""
	Return the upstream commit that our patches (if we have any) are rebased
	on top of.
	"""
	entries = list(reversed(list(get_reflog_entries("HEAD"))))

	for entry in entries:
		# TODO: add more verification that the rebase finished?
		# (and for the right branch)
		if entry["message"].startswith("checkout: moving from "):
			return entry["new"]

	raise RuntimeError(
		"Could not find upstream commit in reflog; "
		"entries are:\n%s" % (pprint.pformat(entries),))

def get_commit(branch_name):
	return subprocess.check_output(["git", "rev-parse", branch_name]).split()[0]

def get_commit_or_none(branch_name):
	try:
		return get_commit(branch_name)
	except subprocess.CalledProcessError, e:
		# fatal: ambiguous argument 'some-branch': unknown revision or
		# path not in the working tree.
		if not 'returned non-zero exit status 128' in str(e):
			raise
		return None

def now():
	return datetime.datetime.now()

def main():
	parser = argparse.ArgumentParser(description="""
	Utility to mark the HEAD with a branch and timestamped tag, and the upstream
	commit (that we're rebased on top of) with another timestamped tag.  All three
	markings are optional.

	This allows you to

	1) easily find an older set of rebased patches
	2) see in tig/gitk which commits you've previously rebased onto.  This is
	   useful for seeing which new commits you might need to review.

	For any of --tag-head, --branch-head, and --tag-upstream, you can use
	{YMDHMS} to insert the current time, or {YMDN} to insert the current date
	with a counter that avoids collision with existing tags.
	""")

	parser.add_argument('-t', '--tag-head', dest='tag_head',
		help="create a tag with this name pointing to HEAD")

	parser.add_argument('-b', '--branch-head', dest='branch_head',
		help="force-create a branch with this name pointing to HEAD")

	parser.add_argument('-u', '--tag-upstream', dest='tag_upstream',
		help="create a tag with this name pointing to the upstream commit")

	args = parser.parse_args()

	if not (args.tag_head or args.branch_head or args.tag_upstream):
		print >>sys.stderr, "Must specify one or more of --tag-head, " \
			"--branch-head, or --tag-upstream; see --help"
		sys.exit(1)

	t = now()

	if args.tag_head or args.tag_upstream:
		upstream_commit = get_upstream_commit()

	if args.tag_upstream:
		existing_tags_on_upstream = get_tags_on_commit(upstream_commit)
		if any(get_re_for_format_string(args.tag_upstream).match(tag)
			 for tag in existing_tags_on_upstream):
			print >>sys.stderr, "Upstream commit %s already has tags " \
				"%r; not adding another tag." % (upstream_commit, existing_tags_on_upstream)
		else:
			call(["git", "tag", "-a", "--message", "",
				get_expanded_name(args.tag_upstream, t),
				upstream_commit])

	if args.tag_head:
		existing_tags_on_head = get_tags_on_commit("HEAD")
		if any(get_re_for_format_string(args.tag_head).match(tag)
			 for tag in existing_tags_on_head):
			print >>sys.stderr, "HEAD already has tags " \
				"%r; not adding another tag." % (existing_tags_on_head,)
		else:
			call(["git", "tag", "-a", "--message",
				get_tag_message(upstream_commit),
				get_expanded_name(args.tag_head, t)])

	if args.branch_head:
		branch_name = get_expanded_name(args.branch_head, t)
		if get_commit_or_none(branch_name) == get_commit("HEAD"):
			print >>sys.stderr, "HEAD is already marked as %s; " \
				"skipping branch -f." % (branch_name,)
		else:
			call(["git", "branch", "-f", branch_name])

if __name__ == '__main__':
	main()
<commit_msg>Remove useless call() wrapper<commit_after>#!/usr/bin/env python

"""
Utility to mark the HEAD with a branch and timestamped tag, and the upstream
commit (that we're rebased on top of) with another timestamped tag.

This allows you to

1) easily find an older set of rebased patches with `git tag`
2) see in tig/gitk which commits you've previously rebased onto.  This is
   useful for seeing which new commits you might need to review.

Sample usage:
git pull --rebase
tagmyrebase.py --branch-head good --tag-head 'good-{YMDN}' --tag-upstream 'U-{YMDN}'

All three arguments are optional.

For any of --tag-head, --branch-head, and --tag-upstream, you can use {YMDHMS}
to insert the current time, or {YMDN} to insert the current date with a
counter that avoids collision with existing tags.
"""

__version__ = '0.2'

import re
import sys
import datetime
import subprocess
import argparse
import pprint

def get_re_for_format_string(format_string):
	"""
	Returns a regular expression for a format string, so that we can check
	if a tag already exists.
	"""
	YMDHMS_RE = r'\d\d\d\d-[01]\d-[0-3]\d_[0-2]\d-[0-5]\d-[0-6]\d'
	YMDN_RE = r'\d\d\d\d-[01]\d-[0-3]\d\.\d+'
	return re.compile(
		r'\A' +
		format_string
			.replace('{YMDHMS}', YMDHMS_RE)
			.replace('{YMDN}', YMDN_RE) +
		r'\Z')

def get_tags_on_commit(commit):
	stdout = subprocess.check_output(["git", "tag", "-l", "--points-at", commit])
	tags = stdout.replace("\r", "").strip("\n").split("\n")
	return tags

def get_all_tags():
	stdout = subprocess.check_output(["git", "tag", "-l"])
	tags = stdout.replace("\r", "").strip("\n").split("\n")
	return tags

def get_expanded_name(format_string, t):
	ymdn = None
	if '{YMDN}' in format_string:
		all_tags = set(get_all_tags())
		ymd = t.strftime('%Y-%m-%d')
		for n in xrange(1, 100000):
			proposed_ymdn = ymd + '.' + str(n)
			proposed_tag = get_expanded_name(
				format_string.format(
					YMDN=proposed_ymdn,
					YMDHMS='{YMDHMS}'
				), t)
			if not proposed_tag in all_tags:
				ymdn = proposed_ymdn
				break
		else:
			raise RuntimeError("100,000 tags in one day is too many tags")

	return format_string.format(
		YMDN=ymdn,
		YMDHMS=t.strftime('%Y-%m-%d_%H-%M-%S')
	)

def get_tag_message(upstream_commit):
	# Annotated tag already has tagger name, e-mail, and date
	# TODO XXX: what if this date doesn't match the date that Python gets?
	# Maybe set GIT_COMMITTER_DATE for git tag
	return "Onto: %s\n" % (upstream_commit,)

def get_reflog_entries(branch_name):
	for line in open(".git/logs/" + branch_name, "rb"):
		before_email, after_email = line.split(">", 1)
		before_email += ">"
		old, new, email = before_email.split(" ", 2)
		_, date, tz = after_email.split("\t", 1)[0].split(" ", 2)
		message = after_email.split("\t", 1)[1]
		assert _ == "", repr(_)
		yield dict(old=old, new=new, email=email, date=date, tz=tz, message=message)

def get_upstream_commit():
	"""
	Return the upstream commit that our patches (if we have any) are rebased
	on top of.
	"""
	entries = list(reversed(list(get_reflog_entries("HEAD"))))

	for entry in entries:
		# TODO: add more verification that the rebase finished?
		# (and for the right branch)
		if entry["message"].startswith("checkout: moving from "):
			return entry["new"]

	raise RuntimeError(
		"Could not find upstream commit in reflog; "
		"entries are:\n%s" % (pprint.pformat(entries),))

def get_commit(branch_name):
	return subprocess.check_output(["git", "rev-parse", branch_name]).split()[0]

def get_commit_or_none(branch_name):
	try:
		return get_commit(branch_name)
	except subprocess.CalledProcessError, e:
		# fatal: ambiguous argument 'some-branch': unknown revision or
		# path not in the working tree.
		if not 'returned non-zero exit status 128' in str(e):
			raise
		return None

def now():
	return datetime.datetime.now()

def main():
	parser = argparse.ArgumentParser(description="""
	Utility to mark the HEAD with a branch and timestamped tag, and the upstream
	commit (that we're rebased on top of) with another timestamped tag.  All three
	markings are optional.

	This allows you to

	1) easily find an older set of rebased patches
	2) see in tig/gitk which commits you've previously rebased onto.  This is
	   useful for seeing which new commits you might need to review.

	For any of --tag-head, --branch-head, and --tag-upstream, you can use
	{YMDHMS} to insert the current time, or {YMDN} to insert the current date
	with a counter that avoids collision with existing tags.
	""")

	parser.add_argument('-t', '--tag-head', dest='tag_head',
		help="create a tag with this name pointing to HEAD")

	parser.add_argument('-b', '--branch-head', dest='branch_head',
		help="force-create a branch with this name pointing to HEAD")

	parser.add_argument('-u', '--tag-upstream', dest='tag_upstream',
		help="create a tag with this name pointing to the upstream commit")

	args = parser.parse_args()

	if not (args.tag_head or args.branch_head or args.tag_upstream):
		print >>sys.stderr, "Must specify one or more of --tag-head, " \
			"--branch-head, or --tag-upstream; see --help"
		sys.exit(1)

	t = now()

	if args.tag_head or args.tag_upstream:
		upstream_commit = get_upstream_commit()

	if args.tag_upstream:
		existing_tags_on_upstream = get_tags_on_commit(upstream_commit)
		if any(get_re_for_format_string(args.tag_upstream).match(tag)
			 for tag in existing_tags_on_upstream):
			print >>sys.stderr, "Upstream commit %s already has tags " \
				"%r; not adding another tag." % (upstream_commit, existing_tags_on_upstream)
		else:
			subprocess.check_call(["git", "tag", "-a", "--message", "",
				get_expanded_name(args.tag_upstream, t),
				upstream_commit])

	if args.tag_head:
		existing_tags_on_head = get_tags_on_commit("HEAD")
		if any(get_re_for_format_string(args.tag_head).match(tag)
			 for tag in existing_tags_on_head):
			print >>sys.stderr, "HEAD already has tags " \
				"%r; not adding another tag." % (existing_tags_on_head,)
		else:
			subprocess.check_call(["git", "tag", "-a", "--message",
				get_tag_message(upstream_commit),
				get_expanded_name(args.tag_head, t)])

	if args.branch_head:
		branch_name = get_expanded_name(args.branch_head, t)
		if get_commit_or_none(branch_name) == get_commit("HEAD"):
			print >>sys.stderr, "HEAD is already marked as %s; " \
				"skipping branch -f." % (branch_name,)
		else:
			subprocess.check_call(["git", "branch", "-f", branch_name])

if __name__ == '__main__':
	main()
<|endoftext|>