<commit_before>
from key import Key

def _object_getattr(obj, field):
  '''Attribute getter for the objects to operate on.

  This function can be overridden in classes or instances of Query, Filter, and
  Order. Thus, a custom function to extract values to attributes can be
  specified, and the system can remain agnostic to the client's data model,
  without loosing query power.

  For example, the default implementation works with attributes and items::

    def _object_getattr(obj, field):
      # check whether this key is an attribute
      if hasattr(obj, field):
        value = getattr(obj, field)

      # if not, perhaps it is an item (raw dicts, etc)
      elif field in obj:
        value = obj[field]

      # return whatever we've got.
      return value

  Or consider a more complex, application-specific structure::

    def _object_getattr(version, field):

      if field in ['key', 'committed', 'created', 'hash']:
        return getattr(version, field)

      else:
        return version.attributes[field]['value']

  '''

  # TODO: consider changing this to raise an exception if no value is found.
  value = None

  # check whether this key is an attribute
  if hasattr(obj, field):
    value = getattr(obj, field)

  # if not, perhaps it is an item (raw dicts, etc)
  elif field in obj:
    value = obj[field]

  # return whatever we've got.
  return value

def limit_gen(limit, iterable):
  '''A generator that applies a count `limit`.'''
  limit = int(limit)
  assert limit >= 0, 'negative limit'

  for item in iterable:
    if limit <= 0:
      break
    yield item
    limit -= 1

def offset_gen(offset, iterable, skip_signal=None):
  '''A generator that applies an `offset`, skipping `offset` elements from
  `iterable`. If skip_signal is a callable, it will be called with every
  skipped element.
  '''
  offset = int(offset)
  assert offset >= 0, 'negative offset'

  for item in iterable:
    if offset > 0:
      offset -= 1
      if callable(skip_signal):
        skip_signal(item)
    else:
      yield item

def chain_gen(iterables):
  '''A generator that chains `iterables`.'''
  for iterable in iterables:
    for item in iterable:
      yield item

class Filter(object):
  '''Represents a Filter for a specific field and its value.

  Filters are used on queries to narrow down the set of matching objects.

  Args:
    field: the attribute name (string) on which to apply the filter.

    op: the conditional operator to apply (one of
        ['<', '<=', '=', '!=', '>=', '>']).

    value: the attribute value to compare against.

  Examples::

    Filter('name', '=', 'John Cleese')
    Filter('age', '>=', 18)

  '''

  conditional_operators = ['<', '<=', '=', '!=', '>=', '>']
  '''Conditional operators that Filters support.'''

  _conditional_cmp = {
    "<"  : lambda a, b: a < b,
    "<=" : lambda a, b: a <= b,
    "="  : lambda a, b: a == b,
    "!=" : lambda a, b: a != b,
    ">=" : lambda a, b: a >= b,
    ">"  : lambda a, b: a > b
  }

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, field, op, value):
    if op not in self.conditional_operators:
      raise ValueError('"%s" is not a valid filter Conditional Operator' % op)

    self.field = field
    self.op = op
    self.value = value

  def __call__(self, obj):
    '''Returns whether this object passes this filter.
    This method aggressively tries to find the appropriate value.
    '''
    value = self.object_getattr(obj, self.field)

    # TODO: which way should the direction go here? it may make more sense to
    #       convert the passed-in value instead. Or try both? Or not at all?
    if not isinstance(value, self.value.__class__) and not self.value is None:
      value = self.value.__class__(value)

    return self.valuePasses(value)

  def valuePasses(self, value):
    '''Returns whether this value passes this filter'''
    return self._conditional_cmp[self.op](value, self.value)

  def __str__(self):
    return '%s %s %s' % (self.field, self.op, self.value)

  def __repr__(self):
    return "Filter('%s', '%s', %s)" % (self.field, self.op, repr(self.value))

  def __eq__(self, o):
    return self.field == o.field and self.op == o.op and self.value == o.value

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def generator(self, iterable):
    '''Generator function that iteratively filters given `items`.'''
    for item in iterable:
      if self(item):
        yield item

  @classmethod
  def filter(cls, filters, iterable):
    '''Returns the elements in `iterable` that pass given `filters`'''
    if isinstance(filters, Filter):
      filters = [filters]

    for filter in filters:
      iterable = filter.generator(iterable)

    return iterable

class Order(object):
  '''Represents an Order upon a specific field, and a direction.
  Orders are used on queries to define how they operate on objects

  Args:
    order: an order in string form. This follows the format: [+-]name
           where + is ascending, - is descending, and name is the name
           of the field to order by.
           Note: if no ordering operator is specified, + is default.

  Examples::

    Order('+name')   #  ascending order by name
    Order('-age')    # descending order by age
    Order('score')   #  ascending order by score

  '''

  order_operators = ['-', '+']
  '''Ordering operators: + is ascending, - is descending.'''

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, order):
    self.op = '+'

    try:
      if order[0] in self.order_operators:
        self.op = order[0]
        order = order[1:]
    except IndexError:
      raise ValueError('Order input be at least two characters long.')

    self.field = order

    if self.op not in self.order_operators:
      raise ValueError('"%s" is not a valid Order Operator.' % op)

  def __str__(self):
    return '%s%s' % (self.op, self.field)

  def __repr__(self):
    return "Order('%s%s')" % (self.op, self.field)

  def __eq__(self, other):
    return self.field == other.field and self.op == other.op

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def isAscending(self):
    return self.op == '+'

  def isDescending(self):
    return not self.isAscending()

  def keyfn(self, obj):
    '''A key function to be used in pythonic sort operations.'''
    return self.object_getattr(obj, self.field)

  @classmethod
  def multipleOrderComparison(cls, orders):
    '''Returns a function that will compare two items according to `orders`'''
    comparers = [ (o.keyfn, 1 if o.isAscending() else -1) for o in orders]

    def cmpfn(a, b):
      for keyfn, ascOrDesc in comparers:
        comparison = cmp(keyfn(a), keyfn(b)) * ascOrDesc
        if comparison is not 0:
          return comparison
      return 0

    return cmpfn

  @classmethod
  def sorted(cls, items, orders):
    '''Returns the elements in `items` sorted according to `orders`'''
    return sorted(items, cmp=cls.multipleOrderComparison(orders))

class Query(object):
  '''A Query describes a set of objects.

  Queries are used to retrieve objects and instances matching a set of criteria
  from Datastores. Query objects themselves are simply descriptions,
  the actual Query implementations are left up to the Datastores.
  '''

  '''Object attribute getter. Can be overridden to match client data model.'''
  object_getattr = staticmethod(_object_getattr)

  def __init__(self, key, limit=None, offset=0, object_getattr=None):
    ''' Initialize a query.

    Parameters
      key: a key representing the level of this query. For example, a Query with
           Key('/MontyPython/Actor:') would query objects in that key path, eg:
             Key('/MontyPython/Actor:JohnCleese')
             Key('/MontyPython/Actor:EricIdle')
             Key('/MontyPython/Actor:GrahamChapman')

            It is up to datastores how to implement this namespacing. E.g.,
            some datastores may store values in different tables or collections.

      limit: an integer representing the maximum number of results to return.

      offset: an integer representing a number of results to skip.

      object_getattr: a function to extract attribute values from an object. It
           is used to satisfy query filters and orders. Defining this function
           allows the client to control the data model of the stored values.
           The default function attempts to access values as attributes
           (__getattr__) or items (__getitem__).
    '''
    if not isinstance(key, Key):
      raise TypeError('key must be of type %s' % Key)

    self.key = key

    self.limit = int(limit) if limit is not None else None
    self.offset = int(offset)

    self.filters = []
    self.orders = []

    if object_getattr:
      self.object_getattr = object_getattr

  def __str__(self):
    '''Returns a string describing this query.'''
    return repr(self)

  def __repr__(self):
    '''Returns the representation of this query. Enables eval(repr(.)).'''
    return 'Query.from_dict(%s)' % self.dict()

  def __call__(self, iterable):
    '''Naively apply this query on an iterable of objects.
    Applying a query applies filters, sorts by appropriate orders, and returns
    a limited set.

    WARNING: When orders are applied, this function operates on the entire set
             of entities directly, not just iterators/generators. That means
             the entire result set will be in memory. Datastores with large
             objects and large query results should translate the Query and
             perform their own optimizations.
    '''

    cursor = Cursor(self, iterable)
    cursor.apply_filter()
    cursor.apply_order()
    cursor.apply_offset()
    cursor.apply_limit()
    return cursor

  def order(self, order):
    '''Adds an Order to this query.

    Args:
      see :py:class:`Order <datastore.query.Order>` constructor

    Returns self for JS-like method chaining::

      query.order('+age').order('-home')

    '''
    order = order if isinstance(order, Order) else Order(order)

    # ensure order gets attr values the same way the rest of the query does.
    order.object_getattr = self.object_getattr
    self.orders.append(order)
    return self # for chaining

  def filter(self, *args):
    '''Adds a Filter to this query.

    Args:
      see :py:class:`Filter <datastore.query.Filter>` constructor

    Returns self for JS-like method chaining::

      query.filter('age', '>', 18).filter('sex', '=', 'Female')

    '''
    if len(args) == 1 and isinstance(args[0], Filter):
      filter = args[0]
    else:
      filter = Filter(*args)

    # ensure filter gets attr values the same way the rest of the query does.
    filter.object_getattr = self.object_getattr
    self.filters.append(filter)
    return self # for chaining

  def __cmp__(self, other):
    return cmp(self.dict(), other.dict())

  def __hash__(self):
    return hash(repr(self))

  def copy(self):
    '''Returns a copy of this query.'''
    if self.object_getattr is Query.object_getattr:
      other = Query(self.key)
    else:
      other = Query(self.key, object_getattr=self.object_getattr)
    other.limit = self.limit
    other.offset = self.offset
    other.filters = self.filters
    other.orders = self.orders
    return other

  def dict(self):
    '''Returns a dictionary representing this query.'''
    d = dict()
    d['key'] = str(self.key)

    if self.limit is not None:
      d['limit'] = self.limit
    if self.offset > 0:
      d['offset'] = self.offset
    if len(self.filters) > 0:
      d['filter'] = [[f.field, f.op, f.value] for f in self.filters]
    if len(self.orders) > 0:
      d['order'] = [str(o) for o in self.orders]

    return d

  @classmethod
  def from_dict(cls, dictionary):
    '''Constructs a query from a dictionary.'''
    query = cls(Key(dictionary['key']))

    for key, value in dictionary.items():

      if key == 'order':
        for order in value:
          query.order(order)

      elif key == 'filter':
        for filter in value:
          if not isinstance(filter, Filter):
            filter = Filter(*filter)
          query.filter(filter)

      elif key in ['limit', 'offset']:
        setattr(query, key, value)
    return query

def is_iterable(obj):
  return hasattr(obj, '__iter__') or hasattr(obj, '__getitem__')

class Cursor(object):
  '''Represents a query result generator.'''

  __slots__ = ('query', '_iterable', '_iterator', 'skipped', 'returned', )

  def __init__(self, query, iterable):
    if not isinstance(query, Query):
      raise ValueError('Cursor received invalid query: %s' % query)

    if not is_iterable(iterable):
      raise ValueError('Cursor received invalid iterable: %s' % iterable)

    self.query = query
    self._iterable = iterable
    self._iterator = None
    self.returned = 0
    self.skipped = 0

  def __iter__(self):
    '''The cursor itself is the iterator. Note that it cannot be used twice,
    and once iteration starts, the cursor cannot be modified.
    '''
    if self._iterator:
      raise RuntimeError('Attempt to iterate over Cursor twice.')

    self._iterator = iter(self._iterable)
    return self

  def next(self):
    '''Iterator next. Build up count of returned elements during iteration.'''

    # if iteration has not begun, begin it.
    if not self._iterator:
      self.__iter__()

    next = self._iterator.next()
    if next is not StopIteration:
      self._returned_inc(next)
    return next

  def _skipped_inc(self, item):
    '''A function to increment the skipped count.'''
    self.skipped += 1

  def _returned_inc(self, item):
    '''A function to increment the returned count.'''
    self.returned += 1

  def _ensure_modification_is_safe(self):
    '''Assertions to ensure modification of this Cursor is safe.'''
    assert self.query, 'Cursor must have a Query.'
    assert is_iterable(self._iterable), 'Cursor must have a resultset iterable.'
    assert not self._iterator, 'Cursor must not be modified after iteration.'

  def apply_filter(self):
    '''Naively apply query filters.'''
    self._ensure_modification_is_safe()

    if len(self.query.filters) > 0:
      self._iterable = Filter.filter(self.query.filters, self._iterable)

  def apply_order(self):
    '''Naively apply query orders.'''
    self._ensure_modification_is_safe()

    if len(self.query.orders) > 0:
      self._iterable = Order.sorted(self._iterable, self.query.orders)
      # not a generator :(

  def apply_offset(self):
    '''Naively apply query offset.'''
    self._ensure_modification_is_safe()

    if self.query.offset != 0:
      self._iterable = \
        offset_gen(self.query.offset, self._iterable, self._skipped_inc)
        # _skipped_inc helps keep count of skipped elements

  def apply_limit(self):
    '''Naively apply query limit.'''
    self._ensure_modification_is_safe()

    if self.query.limit is not None:
      self._iterable = limit_gen(self.query.limit, self._iterable)

<commit_msg>Query / by default<commit_after>
from key import Key

def _object_getattr(obj, field):
  '''Attribute getter for the objects to operate on.

  This function can be overridden in classes or instances of Query, Filter, and
  Order. Thus, a custom function to extract values to attributes can be
  specified, and the system can remain agnostic to the client's data model,
  without loosing query power.

  For example, the default implementation works with attributes and items::

    def _object_getattr(obj, field):
      # check whether this key is an attribute
      if hasattr(obj, field):
        value = getattr(obj, field)

      # if not, perhaps it is an item (raw dicts, etc)
      elif field in obj:
        value = obj[field]

      # return whatever we've got.
      return value

  Or consider a more complex, application-specific structure::

    def _object_getattr(version, field):

      if field in ['key', 'committed', 'created', 'hash']:
        return getattr(version, field)

      else:
        return version.attributes[field]['value']

  '''

  # TODO: consider changing this to raise an exception if no value is found.
  value = None

  # check whether this key is an attribute
  if hasattr(obj, field):
    value = getattr(obj, field)

  # if not, perhaps it is an item (raw dicts, etc)
  elif field in obj:
    value = obj[field]

  # return whatever we've got.
  return value

def limit_gen(limit, iterable):
  '''A generator that applies a count `limit`.'''
  limit = int(limit)
  assert limit >= 0, 'negative limit'

  for item in iterable:
    if limit <= 0:
      break
    yield item
    limit -= 1

def offset_gen(offset, iterable, skip_signal=None):
  '''A generator that applies an `offset`, skipping `offset` elements from
  `iterable`. If skip_signal is a callable, it will be called with every
  skipped element.
  '''
  offset = int(offset)
  assert offset >= 0, 'negative offset'

  for item in iterable:
    if offset > 0:
      offset -= 1
      if callable(skip_signal):
        skip_signal(item)
    else:
      yield item

def chain_gen(iterables):
  '''A generator that chains `iterables`.'''
  for iterable in iterables:
    for item in iterable:
      yield item

class Filter(object):
  '''Represents a Filter for a specific field and its value.

  Filters are used on queries to narrow down the set of matching objects.

  Args:
    field: the attribute name (string) on which to apply the filter.

    op: the conditional operator to apply (one of
        ['<', '<=', '=', '!=', '>=', '>']).

    value: the attribute value to compare against.

  Examples::

    Filter('name', '=', 'John Cleese')
    Filter('age', '>=', 18)

  '''

  conditional_operators = ['<', '<=', '=', '!=', '>=', '>']
  '''Conditional operators that Filters support.'''

  _conditional_cmp = {
    "<"  : lambda a, b: a < b,
    "<=" : lambda a, b: a <= b,
    "="  : lambda a, b: a == b,
    "!=" : lambda a, b: a != b,
    ">=" : lambda a, b: a >= b,
    ">"  : lambda a, b: a > b
  }

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, field, op, value):
    if op not in self.conditional_operators:
      raise ValueError('"%s" is not a valid filter Conditional Operator' % op)

    self.field = field
    self.op = op
    self.value = value

  def __call__(self, obj):
    '''Returns whether this object passes this filter.
    This method aggressively tries to find the appropriate value.
    '''
    value = self.object_getattr(obj, self.field)

    # TODO: which way should the direction go here? it may make more sense to
    #       convert the passed-in value instead. Or try both? Or not at all?
    if not isinstance(value, self.value.__class__) and not self.value is None:
      value = self.value.__class__(value)

    return self.valuePasses(value)

  def valuePasses(self, value):
    '''Returns whether this value passes this filter'''
    return self._conditional_cmp[self.op](value, self.value)

  def __str__(self):
    return '%s %s %s' % (self.field, self.op, self.value)

  def __repr__(self):
    return "Filter('%s', '%s', %s)" % (self.field, self.op, repr(self.value))

  def __eq__(self, o):
    return self.field == o.field and self.op == o.op and self.value == o.value

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def generator(self, iterable):
    '''Generator function that iteratively filters given `items`.'''
    for item in iterable:
      if self(item):
        yield item

  @classmethod
  def filter(cls, filters, iterable):
    '''Returns the elements in `iterable` that pass given `filters`'''
    if isinstance(filters, Filter):
      filters = [filters]

    for filter in filters:
      iterable = filter.generator(iterable)

    return iterable

class Order(object):
  '''Represents an Order upon a specific field, and a direction.
  Orders are used on queries to define how they operate on objects

  Args:
    order: an order in string form. This follows the format: [+-]name
           where + is ascending, - is descending, and name is the name
           of the field to order by.
           Note: if no ordering operator is specified, + is default.

  Examples::

    Order('+name')   #  ascending order by name
    Order('-age')    # descending order by age
    Order('score')   #  ascending order by score

  '''

  order_operators = ['-', '+']
  '''Ordering operators: + is ascending, - is descending.'''

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, order):
    self.op = '+'

    try:
      if order[0] in self.order_operators:
        self.op = order[0]
        order = order[1:]
    except IndexError:
      raise ValueError('Order input be at least two characters long.')

    self.field = order

    if self.op not in self.order_operators:
      raise ValueError('"%s" is not a valid Order Operator.' % op)

  def __str__(self):
    return '%s%s' % (self.op, self.field)

  def __repr__(self):
    return "Order('%s%s')" % (self.op, self.field)

  def __eq__(self, other):
    return self.field == other.field and self.op == other.op

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def isAscending(self):
    return self.op == '+'

  def isDescending(self):
    return not self.isAscending()

  def keyfn(self, obj):
    '''A key function to be used in pythonic sort operations.'''
    return self.object_getattr(obj, self.field)

  @classmethod
  def multipleOrderComparison(cls, orders):
    '''Returns a function that will compare two items according to `orders`'''
    comparers = [ (o.keyfn, 1 if o.isAscending() else -1) for o in orders]

    def cmpfn(a, b):
      for keyfn, ascOrDesc in comparers:
        comparison = cmp(keyfn(a), keyfn(b)) * ascOrDesc
        if comparison is not 0:
          return comparison
      return 0

    return cmpfn

  @classmethod
  def sorted(cls, items, orders):
    '''Returns the elements in `items` sorted according to `orders`'''
    return sorted(items, cmp=cls.multipleOrderComparison(orders))

class Query(object):
  '''A Query describes a set of objects.

  Queries are used to retrieve objects and instances matching a set of criteria
  from Datastores. Query objects themselves are simply descriptions,
  the actual Query implementations are left up to the Datastores.
  '''

  '''Object attribute getter. Can be overridden to match client data model.'''
  object_getattr = staticmethod(_object_getattr)

  def __init__(self, key=Key('/'), limit=None, offset=0, object_getattr=None):
    ''' Initialize a query.

    Parameters
      key: a key representing the level of this query. For example, a Query with
           Key('/MontyPython/Actor:') would query objects in that key path, eg:
             Key('/MontyPython/Actor:JohnCleese')
             Key('/MontyPython/Actor:EricIdle')
             Key('/MontyPython/Actor:GrahamChapman')

            It is up to datastores how to implement this namespacing. E.g.,
            some datastores may store values in different tables or collections.

      limit: an integer representing the maximum number of results to return.

      offset: an integer representing a number of results to skip.

      object_getattr: a function to extract attribute values from an object. It
           is used to satisfy query filters and orders. Defining this function
           allows the client to control the data model of the stored values.
           The default function attempts to access values as attributes
           (__getattr__) or items (__getitem__).
    '''
    if not isinstance(key, Key):
      raise TypeError('key must be of type %s' % Key)

    self.key = key

    self.limit = int(limit) if limit is not None else None
    self.offset = int(offset)

    self.filters = []
    self.orders = []

    if object_getattr:
      self.object_getattr = object_getattr

  def __str__(self):
    '''Returns a string describing this query.'''
    return repr(self)

  def __repr__(self):
    '''Returns the representation of this query. Enables eval(repr(.)).'''
    return 'Query.from_dict(%s)' % self.dict()

  def __call__(self, iterable):
    '''Naively apply this query on an iterable of objects.
    Applying a query applies filters, sorts by appropriate orders, and returns
    a limited set.

    WARNING: When orders are applied, this function operates on the entire set
             of entities directly, not just iterators/generators. That means
             the entire result set will be in memory. Datastores with large
             objects and large query results should translate the Query and
             perform their own optimizations.
    '''

    cursor = Cursor(self, iterable)
    cursor.apply_filter()
    cursor.apply_order()
    cursor.apply_offset()
    cursor.apply_limit()
    return cursor

  def order(self, order):
    '''Adds an Order to this query.

    Args:
      see :py:class:`Order <datastore.query.Order>` constructor

    Returns self for JS-like method chaining::

      query.order('+age').order('-home')

    '''
    order = order if isinstance(order, Order) else Order(order)

    # ensure order gets attr values the same way the rest of the query does.
    order.object_getattr = self.object_getattr
    self.orders.append(order)
    return self # for chaining

  def filter(self, *args):
    '''Adds a Filter to this query.

    Args:
      see :py:class:`Filter <datastore.query.Filter>` constructor

    Returns self for JS-like method chaining::

      query.filter('age', '>', 18).filter('sex', '=', 'Female')

    '''
    if len(args) == 1 and isinstance(args[0], Filter):
      filter = args[0]
    else:
      filter = Filter(*args)

    # ensure filter gets attr values the same way the rest of the query does.
    filter.object_getattr = self.object_getattr
    self.filters.append(filter)
    return self # for chaining

  def __cmp__(self, other):
    return cmp(self.dict(), other.dict())

  def __hash__(self):
    return hash(repr(self))

  def copy(self):
    '''Returns a copy of this query.'''
    if self.object_getattr is Query.object_getattr:
      other = Query(self.key)
    else:
      other = Query(self.key, object_getattr=self.object_getattr)
    other.limit = self.limit
    other.offset = self.offset
    other.filters = self.filters
    other.orders = self.orders
    return other

  def dict(self):
    '''Returns a dictionary representing this query.'''
    d = dict()
    d['key'] = str(self.key)

    if self.limit is not None:
      d['limit'] = self.limit
    if self.offset > 0:
      d['offset'] = self.offset
    if len(self.filters) > 0:
      d['filter'] = [[f.field, f.op, f.value] for f in self.filters]
    if len(self.orders) > 0:
      d['order'] = [str(o) for o in self.orders]

    return d

  @classmethod
  def from_dict(cls, dictionary):
    '''Constructs a query from a dictionary.'''
    query = cls(Key(dictionary['key']))

    for key, value in dictionary.items():

      if key == 'order':
        for order in value:
          query.order(order)

      elif key == 'filter':
        for filter in value:
          if not isinstance(filter, Filter):
            filter = Filter(*filter)
          query.filter(filter)

      elif key in ['limit', 'offset']:
        setattr(query, key, value)
    return query

def is_iterable(obj):
  return hasattr(obj, '__iter__') or hasattr(obj, '__getitem__')

class Cursor(object):
  '''Represents a query result generator.'''

  __slots__ = ('query', '_iterable', '_iterator', 'skipped', 'returned', )

  def __init__(self, query, iterable):
    if not isinstance(query, Query):
      raise ValueError('Cursor received invalid query: %s' % query)

    if not is_iterable(iterable):
      raise ValueError('Cursor received invalid iterable: %s' % iterable)

    self.query = query
    self._iterable = iterable
    self._iterator = None
    self.returned = 0
    self.skipped = 0

  def __iter__(self):
    '''The cursor itself is the iterator. Note that it cannot be used twice,
    and once iteration starts, the cursor cannot be modified.
    '''
    if self._iterator:
      raise RuntimeError('Attempt to iterate over Cursor twice.')

    self._iterator = iter(self._iterable)
    return self

  def next(self):
    '''Iterator next. Build up count of returned elements during iteration.'''

    # if iteration has not begun, begin it.
    if not self._iterator:
      self.__iter__()

    next = self._iterator.next()
    if next is not StopIteration:
      self._returned_inc(next)
    return next

  def _skipped_inc(self, item):
    '''A function to increment the skipped count.'''
    self.skipped += 1

  def _returned_inc(self, item):
    '''A function to increment the returned count.'''
    self.returned += 1

  def _ensure_modification_is_safe(self):
    '''Assertions to ensure modification of this Cursor is safe.'''
    assert self.query, 'Cursor must have a Query.'
    assert is_iterable(self._iterable), 'Cursor must have a resultset iterable.'
    assert not self._iterator, 'Cursor must not be modified after iteration.'

  def apply_filter(self):
    '''Naively apply query filters.'''
    self._ensure_modification_is_safe()

    if len(self.query.filters) > 0:
      self._iterable = Filter.filter(self.query.filters, self._iterable)

  def apply_order(self):
    '''Naively apply query orders.'''
    self._ensure_modification_is_safe()

    if len(self.query.orders) > 0:
      self._iterable = Order.sorted(self._iterable, self.query.orders)
      # not a generator :(

  def apply_offset(self):
    '''Naively apply query offset.'''
    self._ensure_modification_is_safe()

    if self.query.offset != 0:
      self._iterable = \
        offset_gen(self.query.offset, self._iterable, self._skipped_inc)
        # _skipped_inc helps keep count of skipped elements

  def apply_limit(self):
    '''Naively apply query limit.'''
    self._ensure_modification_is_safe()

    if self.query.limit is not None:
      self._iterable = limit_gen(self.query.limit, self._iterable)

<|endoftext|>


<commit_before>from dataviva import db
from dataviva.utils.auto_serialize import AutoSerialize
from dataviva.attrs.models import Bra, Cnae, Cbo

from sqlalchemy import and_

############################################################
# ----------------------------------------------------------
# 2 variable tables
#
############################################################

class BaseRais(object):
    year = db.Column(db.Integer(4), primary_key=True)

    wage = db.Column(db.Numeric(16,2))
    num_emp = db.Column(db.Integer(11))
    num_jobs = db.Column(db.Integer(11))
    num_est = db.Column(db.Integer(11))
    wage_avg = db.Column(db.Numeric(16,2))
    age_avg = db.Column(db.Numeric(16,2))

    wage_growth = db.Column(db.Float())
    wage_growth_5 = db.Column(db.Float())

    num_emp_growth = db.Column(db.Float())
    num_emp_growth_5 = db.Column(db.Float())

class Yi(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yi'
    year = db.Column(db.Integer(4), primary_key=True)
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)

    cbo_diversity = db.Column(db.Integer(11))
    cbo_diversity_eff = db.Column(db.Float())
    bra_diversity = db.Column(db.Integer(11))
    bra_diversity_eff = db.Column(db.Float())

    cnae_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yi %d.%s>' % (self.year, self.cnae_id)

class Yb_rais(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yb'
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)

    cnae_diversity = db.Column(db.Integer(11))
    cnae_diversity_eff = db.Column(db.Float())
    cbo_diversity = db.Column(db.Integer(11))
    cbo_diversity_eff = db.Column(db.Float())

    bra_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yb_rais %d.%s>' % (self.year, self.bra_id)

class Yo(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yo'
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)

    cnae_diversity = db.Column(db.Integer(11))
    cnae_diversity_eff = db.Column(db.Float())
    bra_diversity = db.Column(db.Integer(11))
    bra_diversity_eff = db.Column(db.Float())

    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yo %d.%s>' % (self.year, self.cbo_id)

############################################################
# ----------------------------------------------------------
# 3 variable tables
#
############################################################

class Ybi(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_ybi'
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)

    rca = db.Column(db.Float())
    distance = db.Column(db.Float())
    opp_gain = db.Column(db.Float())

    bra_id_len = db.Column(db.Integer(1))
    cnae_id_len = db.Column(db.Integer(1))

    required_bras = db.Column(db.PickleType)

    def __repr__(self):
        return '<Ybi %d.%s.%s>' % (self.year, self.bra_id, self.cnae_id)

class Ybo(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_ybo'
    year = db.Column(db.Integer(4), primary_key=True)
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)

    bra_id_len = db.Column(db.Integer(1))
    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Ybo %d.%s.%s>' % (self.year, self.bra_id, self.cbo_id)

class Yio(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yio'
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)
    importance = db.Column(db.Float())

    cnae_id_len = db.Column(db.Integer(1))
    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yio %d.%s.%s>' % (self.year, self.cnae_id, self.cbo_id)

class Ybio(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_ybio'
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)
    required = db.Column(db.Float())

    bra_id_len = db.Column(db.Integer(1))
    cnae_id_len = db.Column(db.Integer(1))
    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Ybio %d.%s.%s.%s>' % (self.year, self.bra_id, self.cnae_id, self.cbo_id)
<commit_msg>temporarily removing required_bras<commit_after>from dataviva import db
from dataviva.utils.auto_serialize import AutoSerialize
from dataviva.attrs.models import Bra, Cnae, Cbo

from sqlalchemy import and_

############################################################
# ----------------------------------------------------------
# 2 variable tables
#
############################################################

class BaseRais(object):
    year = db.Column(db.Integer(4), primary_key=True)

    wage = db.Column(db.Numeric(16,2))
    num_emp = db.Column(db.Integer(11))
    num_jobs = db.Column(db.Integer(11))
    num_est = db.Column(db.Integer(11))
    wage_avg = db.Column(db.Numeric(16,2))
    age_avg = db.Column(db.Numeric(16,2))

    wage_growth = db.Column(db.Float())
    wage_growth_5 = db.Column(db.Float())

    num_emp_growth = db.Column(db.Float())
    num_emp_growth_5 = db.Column(db.Float())

class Yi(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yi'
    year = db.Column(db.Integer(4), primary_key=True)
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)

    cbo_diversity = db.Column(db.Integer(11))
    cbo_diversity_eff = db.Column(db.Float())
    bra_diversity = db.Column(db.Integer(11))
    bra_diversity_eff = db.Column(db.Float())

    cnae_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yi %d.%s>' % (self.year, self.cnae_id)

class Yb_rais(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yb'
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)

    cnae_diversity = db.Column(db.Integer(11))
    cnae_diversity_eff = db.Column(db.Float())
    cbo_diversity = db.Column(db.Integer(11))
    cbo_diversity_eff = db.Column(db.Float())

    bra_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yb_rais %d.%s>' % (self.year, self.bra_id)

class Yo(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yo'
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)

    cnae_diversity = db.Column(db.Integer(11))
    cnae_diversity_eff = db.Column(db.Float())
    bra_diversity = db.Column(db.Integer(11))
    bra_diversity_eff = db.Column(db.Float())

    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yo %d.%s>' % (self.year, self.cbo_id)

############################################################
# ----------------------------------------------------------
# 3 variable tables
#
############################################################

class Ybi(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_ybi'
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)

    rca = db.Column(db.Float())
    distance = db.Column(db.Float())
    opp_gain = db.Column(db.Float())

    bra_id_len = db.Column(db.Integer(1))
    cnae_id_len = db.Column(db.Integer(1))

    # required_bras = db.Column(db.PickleType)

    def __repr__(self):
        return '<Ybi %d.%s.%s>' % (self.year, self.bra_id, self.cnae_id)

class Ybo(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_ybo'
    year = db.Column(db.Integer(4), primary_key=True)
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)

    bra_id_len = db.Column(db.Integer(1))
    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Ybo %d.%s.%s>' % (self.year, self.bra_id, self.cbo_id)

class Yio(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_yio'
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)
    importance = db.Column(db.Float())

    cnae_id_len = db.Column(db.Integer(1))
    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Yio %d.%s.%s>' % (self.year, self.cnae_id, self.cbo_id)

class Ybio(BaseRais, db.Model, AutoSerialize):

    __tablename__ = 'rais_ybio'
    bra_id = db.Column(db.String(8), db.ForeignKey(Bra.id), primary_key=True)
    cnae_id = db.Column(db.String(5), db.ForeignKey(Cnae.id), primary_key=True)
    cbo_id = db.Column(db.String(6), db.ForeignKey(Cbo.id), primary_key=True)
    required = db.Column(db.Float())

    bra_id_len = db.Column(db.Integer(1))
    cnae_id_len = db.Column(db.Integer(1))
    cbo_id_len = db.Column(db.Integer(1))

    def __repr__(self):
        return '<Ybio %d.%s.%s.%s>' % (self.year, self.bra_id, self.cnae_id, self.cbo_id)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""dave_navarros_goatee

This attempts to discover potential passwords leaked to the filesystem
via chat and program logs, shell histories, configuration files, and
anything else that is readable.

This is currently very terrible and shouldnt be used.
2/2017 -- Daniel Roberson

TODO:
- add more hash types
-- $id$salt$hash
-- ids: 1=md5 2a=blowfish 5=sha256 6=sha512
- LRU cache to minimize duplicate password attempts
- different pattern for open()
- flag to generate wordlist only
- flag to toggle whitespace in passwords

The MIT License

Copyright (c) 2017 Daniel Roberson

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import os
import sys
import stat
import string
import argparse
import math
import crypt

# Globals
HASHLIST = {}

# Constants
ALPHAONLY = string.ascii_letters
ALPHANUM = ALPHAONLY + string.digits
ALLCHARS = ALPHANUM + string.punctuation

def is_binary(filename):
    """Determine if a file is a binary"""
    binary = open(filename, 'rb')
    chunk = ''
    try:
        while 1:
            try:
                chunk = binary.read(1024)
            except Exception:
                return False
            if '\0' in chunk:
                return True
            if len(chunk) < 1024:
                break
    finally:
        binary.close()
    return False

def shannon_entropy(data, charset):
    """Calculate entropy of a string using the Shannon Algorithm

    Claude Shannon looks like he could have commanded the Death Star.
    - Michael Roberson

    https://en.wikipedia.org/wiki/Claude_Shannon

    Jacked this function from:
    http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html
    """
    if not data:
        return 0
    entropy = 0
    for byte in (ord(c) for c in charset):
        p_x = float(data.count(chr(byte))) / len(data)
        if p_x > 0:
            entropy += - p_x * math.log(p_x, 2)
    return entropy

def analyze(filename, minlength, entropy, charset):
    """Try to find passwords in a file"""
    count = 0
    wordlist = []
    filep = open(filename, 'r')
    print "[-] Analyzing %s" % (os.path.abspath(filename))
    for line in filep:
        if len(HASHLIST.keys()) == 0: # all hashes solved!
            break
        if shannon_entropy(line.rstrip('\r\n'), charset) < entropy:
            continue
        wordlist = mutate(line.rstrip('\r\n'))
        for word in wordlist:
            if len(HASHLIST.keys()) == 0: # all hashes solved!
                break
            if len(word) < minlength:
                continue
            for user in HASHLIST.keys():
                crypted = crypt.crypt(word, HASHLIST[user])
                if crypted == HASHLIST[user]:
                    del HASHLIST[user]
                    print "[*] Found password for %s: %s in %s" % \
                        (user, word, os.path.abspath(filename))
                    continue
            count += 1
    print "[-] %s words attempted from %s" % \
        (str(count), os.path.abspath(filename))
    print
    filep.close()

def left_right_substrings(buf):
    """Return substrings containing leftmost and rightmost characters.

    For example, abc' returns:
    ['a', 'ab', 'abc', 'c', 'cb']

    Thanks to Michael Roberson for this function!
    """
    length = len(buf)
    left_right = [buf[0:i+1] for i in xrange(length)]
    right_left = [buf[-i:] for i in xrange(length)]
    return list(set(left_right + right_left))

def mutate(buf):
    """Generate a list of mutations from a buffer"""
    wordlist = [buf]
    wordlist += left_right_substrings(buf)

    tokens = ''.join(set(buf))

    for omit_token in ALPHANUM:
        tokens = tokens.replace(omit_token, '')

    for token in tokens:
        wordlist.extend(buf.split(token)) # add token itself
        for sub_token in buf.split(token):
            wordlist += left_right_substrings(sub_token)

    # return unique list
    wordlist = list(set(wordlist))
    return wordlist

def should_analyze(filename):
    """Determine if filename is worth exploring"""
    try:
        filep = open(filename, 'r')
        filep.close()
    except Exception:
        return False
    return os.access(filename, os.R_OK) \
        and stat.S_ISREG(os.stat(filename).st_mode) \
        and not os.path.islink(filename) \
        and not is_binary(filename)

def populate_hashes(hashfile):
    """Parse a hash file in user:hash format"""
    global HASHLIST
    HASHLIST = {}

    if hashfile == '':
        print "[-] Must specify a hash file with -f/--file"
        sys.exit(os.EX_USAGE)
    try:
        hashp = open(hashfile, 'r')
    except Exception, err:
        print "[-] Could not open hashfile: %s" % (str(err))
        sys.exit(os.EX_USAGE)

    print "[+] Populating hash list from %s" % (hashfile)

    for line in hashp:
        tmp = line.split(':')

        if len(tmp) <= 1:
            continue
        # remove carriage returns and newlines
        tmp[0] = tmp[0].rstrip('\r\n')
        tmp[1] = tmp[1].rstrip('\r\n')
        line = line.rstrip('\r\n')

        if len(tmp) < 2 or tmp[0] == '' or tmp[1] == '':
            print "[-] Skipping %s due to missing fields" % (line)
            continue

        if len(tmp[1]) < 12:
            print "[-] Skipping %s because its not a valid hash" % (line)
            continue

        HASHLIST[tmp[0]] = tmp[1]

    hashp.close()
    if HASHLIST == {}:
        print "[-] Empty hash list. Exiting"
        sys.exit(os.EX_USAGE)

def main():
    """dave_navarros_goatee.py entry point"""

    print "[+] dave_navarros_goatee.py -- by Daniel Roberson"
    print

    # parse CLI arguments
    description = "example: ./dave_navarros_goatee.py -p /home -f hashes.txt"
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument("-p",
                        "--path",
                        default="/",
                        help="starting filesystem location")
    parser.add_argument("-f",
                        "--hashfile",
                        default="",
                        help="file containing hashes in user:hash format")
    parser.add_argument("-e",
                        "--entropy",
                        type=float,
                        default=2.0,
                        help="minimum Shannon entropy")
    parser.add_argument("-m",
                        "--minlength",
                        type=int,
                        default=6,
                        help="minimum password length")
    parser.add_argument("-c",
                        "--charset",
                        choices=["ALL", "ALPHA", "ALPHANUM"],
                        default="ALL",
                        help="character set to use for entropy check")
    args = parser.parse_args()

    charsets = {"ALL": ALLCHARS, "ALPHA": ALPHAONLY, "ALPHANUM": ALPHANUM}
    charset = charsets[args.charset]

    if not os.path.isdir(args.path):
        print "[-] %s is not a directory. exiting." % (args.path)
        sys.exit(os.EX_USAGE)

    # parse hash file
    populate_hashes(args.hashfile)

    print
    print "[+] Walking filesystem starting at %s" % (args.path)
    print "[+] Press Control-C to stop the violence."
    print

    # I don't care about directories here, therefore _
    for root, _, files in os.walk(args.path):
        for filename in files:
            try_file = os.path.join(root, filename)
            if should_analyze(try_file) and len(HASHLIST):
                analyze(try_file, args.minlength, args.entropy, charset)

    print "[+] The last Metroid is in captivity. The galaxy is at peace."

if __name__ == "__main__":
    main()
<commit_msg>at least 2 spaces between inline comments<commit_after>#!/usr/bin/env python

"""dave_navarros_goatee

This attempts to discover potential passwords leaked to the filesystem
via chat and program logs, shell histories, configuration files, and
anything else that is readable.

This is currently very terrible and shouldnt be used.
2/2017 -- Daniel Roberson

TODO:
- add more hash types
-- $id$salt$hash
-- ids: 1=md5 2a=blowfish 5=sha256 6=sha512
- LRU cache to minimize duplicate password attempts
- different pattern for open()
- flag to generate wordlist only
- flag to toggle whitespace in passwords

The MIT License

Copyright (c) 2017 Daniel Roberson

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import os
import sys
import stat
import string
import argparse
import math
import crypt

# Globals
HASHLIST = {}

# Constants
ALPHAONLY = string.ascii_letters
ALPHANUM = ALPHAONLY + string.digits
ALLCHARS = ALPHANUM + string.punctuation

def is_binary(filename):
    """Determine if a file is a binary"""
    binary = open(filename, 'rb')
    chunk = ''
    try:
        while 1:
            try:
                chunk = binary.read(1024)
            except Exception:
                return False
            if '\0' in chunk:
                return True
            if len(chunk) < 1024:
                break
    finally:
        binary.close()
    return False

def shannon_entropy(data, charset):
    """Calculate entropy of a string using the Shannon Algorithm

    Claude Shannon looks like he could have commanded the Death Star.
    - Michael Roberson

    https://en.wikipedia.org/wiki/Claude_Shannon

    Jacked this function from:
    http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html
    """
    if not data:
        return 0
    entropy = 0
    for byte in (ord(c) for c in charset):
        p_x = float(data.count(chr(byte))) / len(data)
        if p_x > 0:
            entropy += - p_x * math.log(p_x, 2)
    return entropy

def analyze(filename, minlength, entropy, charset):
    """Try to find passwords in a file"""
    count = 0
    wordlist = []
    filep = open(filename, 'r')
    print "[-] Analyzing %s" % (os.path.abspath(filename))
    for line in filep:
        if len(HASHLIST.keys()) == 0:  # all hashes solved!
            break
        if shannon_entropy(line.rstrip('\r\n'), charset) < entropy:
            continue
        wordlist = mutate(line.rstrip('\r\n'))
        for word in wordlist:
            if len(HASHLIST.keys()) == 0:  # all hashes solved!
                break
            if len(word) < minlength:
                continue
            for user in HASHLIST.keys():
                crypted = crypt.crypt(word, HASHLIST[user])
                if crypted == HASHLIST[user]:
                    del HASHLIST[user]
                    print "[*] Found password for %s: %s in %s" % \
                        (user, word, os.path.abspath(filename))
                    continue
            count += 1
    print "[-] %s words attempted from %s" % \
        (str(count), os.path.abspath(filename))
    print
    filep.close()

def left_right_substrings(buf):
    """Return substrings containing leftmost and rightmost characters.

    For example, abc' returns:
    ['a', 'ab', 'abc', 'c', 'cb']

    Thanks to Michael Roberson for this function!
    """
    length = len(buf)
    left_right = [buf[0:i+1] for i in xrange(length)]
    right_left = [buf[-i:] for i in xrange(length)]
    return list(set(left_right + right_left))

def mutate(buf):
    """Generate a list of mutations from a buffer"""
    wordlist = [buf]
    wordlist += left_right_substrings(buf)

    tokens = ''.join(set(buf))

    # don't use alphanumeric characters as tokens
    for omit_token in ALPHANUM:
        tokens = tokens.replace(omit_token, '')

    for token in tokens:
        wordlist.extend(buf.split(token))  # add token itself
        for sub_token in buf.split(token):
            wordlist += left_right_substrings(sub_token)

    # return unique list
    return list(set(wordlist))

def should_analyze(filename):
    """Determine if filename is worth exploring"""
    try:
        filep = open(filename, 'r')
        filep.close()
    except Exception:
        return False
    return os.access(filename, os.R_OK) \
        and stat.S_ISREG(os.stat(filename).st_mode) \
        and not os.path.islink(filename) \
        and not is_binary(filename)

def populate_hashes(hashfile):
    """Parse a hash file in user:hash format"""
    global HASHLIST
    HASHLIST = {}

    if hashfile == '':
        print "[-] Must specify a hash file with -f/--file"
        sys.exit(os.EX_USAGE)
    try:
        hashp = open(hashfile, 'r')
    except Exception, err:
        print "[-] Could not open hashfile: %s" % (str(err))
        sys.exit(os.EX_USAGE)

    print "[+] Populating hash list from %s" % (hashfile)

    for line in hashp:
        tmp = line.split(':')

        if len(tmp) <= 1:
            continue
        # remove carriage returns and newlines
        tmp[0] = tmp[0].rstrip('\r\n')
        tmp[1] = tmp[1].rstrip('\r\n')
        line = line.rstrip('\r\n')

        if len(tmp) < 2 or tmp[0] == '' or tmp[1] == '':
            print "[-] Skipping %s due to missing fields" % (line)
            continue

        if len(tmp[1]) < 12:
            print "[-] Skipping %s because its not a valid hash" % (line)
            continue

        HASHLIST[tmp[0]] = tmp[1]

    hashp.close()
    if HASHLIST == {}:
        print "[-] Empty hash list. Exiting"
        sys.exit(os.EX_USAGE)

def main():
    """dave_navarros_goatee.py entry point"""

    print "[+] dave_navarros_goatee.py -- by Daniel Roberson"
    print

    # parse CLI arguments
    description = "example: ./dave_navarros_goatee.py -p /home -f hashes.txt"
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument("-p",
                        "--path",
                        default="/",
                        help="starting filesystem location")
    parser.add_argument("-f",
                        "--hashfile",
                        default="",
                        help="file containing hashes in user:hash format")
    parser.add_argument("-e",
                        "--entropy",
                        type=float,
                        default=2.0,
                        help="minimum Shannon entropy")
    parser.add_argument("-m",
                        "--minlength",
                        type=int,
                        default=6,
                        help="minimum password length")
    parser.add_argument("-c",
                        "--charset",
                        choices=["ALL", "ALPHA", "ALPHANUM"],
                        default="ALL",
                        help="character set to use for entropy check")
    args = parser.parse_args()

    charsets = {"ALL": ALLCHARS, "ALPHA": ALPHAONLY, "ALPHANUM": ALPHANUM}
    charset = charsets[args.charset]

    if not os.path.isdir(args.path):
        print "[-] %s is not a directory. exiting." % (args.path)
        sys.exit(os.EX_USAGE)

    # parse hash file
    populate_hashes(args.hashfile)

    print
    print "[+] Walking filesystem starting at %s" % (args.path)
    print "[+] Press Control-C to stop the violence."
    print

    # I don't care about directories here, therefore _
    for root, _, files in os.walk(args.path):
        for filename in files:
            try_file = os.path.join(root, filename)
            if should_analyze(try_file) and len(HASHLIST):
                analyze(try_file, args.minlength, args.entropy, charset)

    print "[+] The last Metroid is in captivity. The galaxy is at peace."

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import re
from datetime import datetime, timedelta
from math import ceil

from pytz import all_timezones, timezone

HOUR = 3600

def get_tz_offsets():
    tz_offsets = {}
    for tz in all_timezones:
        if hasattr(timezone(tz), '_transition_info'):
            for timezone_info in timezone(tz)._transition_info:
                tz_offsets[re.compile(r'\b%s$' % timezone_info[2])] = timezone_info[0]
    return tz_offsets

def pop_tz_offset_from_string(date_string):
    for timezone_re, offset in _tz_offsets.iteritems():
        if timezone_re.search(date_string):
            date_string = timezone_re.sub('', date_string).rstrip()
            return date_string, offset
    else:
        return date_string, None

def convert_to_local_tz(datetime_obj, datetime_tz_offset):
    return datetime_obj - datetime_tz_offset + local_tz_offset

def get_local_tz_offset():
    delta = datetime.now() - datetime.utcnow()
    days, seconds, microseconds = delta.days, delta.seconds, delta.microseconds
    return timedelta(seconds=days * 24 * HOUR + seconds + ceil(microseconds / 1000000.0))

_tz_offsets = get_tz_offsets()
local_tz_offset = get_local_tz_offset()
<commit_msg>not working<commit_after># -*- coding: utf-8 -*-
import re
from datetime import datetime, timedelta
from math import ceil

from pytz import all_timezones, timezone

HOUR = 3600

def get_tz_offsets():
    transition_info = [timezone(tz)._transition_info for tz in all_timezones if hasattr(timezone(tz), '_transition_info')]
    infos = set((info[2], info[0]) for infolist in transition_info for info in infolist)
    return {re.compile(r'\b%s$' % tz_info[0]): tz_info[1] for tz_info in infos}

def pop_tz_offset_from_string(date_string):
    for timezone_re, offset in _tz_offsets.iteritems():
        if timezone_re.search(date_string):
            date_string = timezone_re.sub('', date_string).rstrip()
            return date_string, offset
    else:
        return date_string, None

def convert_to_local_tz(datetime_obj, datetime_tz_offset):
    return datetime_obj - datetime_tz_offset + local_tz_offset

def get_local_tz_offset():
    delta = datetime.now() - datetime.utcnow()
    days, seconds, microseconds = delta.days, delta.seconds, delta.microseconds
    return timedelta(seconds=days * 24 * HOUR + seconds + ceil(microseconds / 1000000.0))

_tz_offsets = get_tz_offsets()
local_tz_offset = get_local_tz_offset()
<|endoftext|>


<commit_before>"""
Django settings for datavis project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# formerly datavis/settings.py

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os, socket, getpass

BASE_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

#
# FIXME: move secret key out of source code / don't hard-code
#

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'abc-123-this-value-is-totally-insecure'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

VAR_RUN_DIR = BASE_DIR
VAR_LIB_DIR = BASE_DIR
LOG_DIR = os.path.join(BASE_DIR, 'log')

# ALLOWED_HOSTS are the server's IP names, NOT the names of allowed client hosts
# (seems like an unfortunate variable name).
# You may see log errors such as:
# Invalid HTTP_HOST header: 'www.baidu.com'. You may need to add 'www.baidu.com' to ALLOWED_HOSTS.
#
# Don't add those external host names to ALLOWED_HOSTS!
# Hacked sites may have a link to this site, but as I understand it, the redirect
# may contain an HTTP packet with an altered HTTP_HOST and SERVER_NAME, hoping that
# a dumb server, thinking HTTP_HOST is itself, will use it in its own redirects and
# <script> statemtents.  The latter could result in an import of hacker code on a
# client's browser. Setting ALLOWED_HOSTS to the various names for datavis will
# result in packets being ignored if they contain other than the following:
ALLOWED_HOSTS = ['datavis', 'datavis.eol.ucar.edu', 'localhost', '128.117.82.210']

# Application definition

# In django, an app is: "a Python package that is specifically intended
# for use in a Django project".
INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # 'django.contrib.formtools',
    'datetimewidget',
    'timezone_field',
    'ncharts',
)

MIDDLEWARE_CLASSES = (
    # https://docs.djangoproject.com/en/1.7/topics/cache/#order-of-middleware-classes
    # UpdateCacheMiddleware must appear before SessionMiddleware,
    # and LocaleMiddleware FetchFromCacheMiddleware must occur after them.
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'datavis.middleware.InternalUseOnlyMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',
)

# TEMPLATE_DEBUG = True
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, 'templates')
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
                # list if you haven't customized them:
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

ROOT_URLCONF = 'datavis.urls'

WSGI_APPLICATION = 'datavis.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(VAR_LIB_DIR, 'db.sqlite3'),
        'OPTIONS': {'timeout': 20,},
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'US/Mountain'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

# URL to use when referring to static files located in STATIC_ROOT
STATIC_URL = '/static/'

# STATIC_ROOT is where "pythyon3 manage.py collectstatic" puts
# the static files it finds.
# See /etc/httpd/conf/vhosts/datavis.conf:
#	Alias /static/ /var/django/eol-django-datavis/static/
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

# People who should receive emails of ERRORs
ADMINS = (
    ('Gordon Maclean', 'maclean@ucar.edu'),
    # ('Hien Nguyen', 'hnguyen@ucar.edu'),
)
EMAIL_HOST = "smtp.eol.ucar.edu"
# Email address they appear to come from
SERVER_EMAIL = getpass.getuser() + '@' + socket.getfqdn()

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format' : "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
            'datefmt' : "%d/%b/%Y %H:%M:%S"
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'django_debug': {
            'level': 'DEBUG',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'django_debug.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'django': {
            'level': 'WARNING',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'django.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'datavis_debug': {
            'level': 'DEBUG',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'datavis_debug.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'datavis': {
            'level': 'WARNING',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'datavis.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'ncharts': {
            'level': 'WARNING',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'ncharts.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'ncharts_debug': {
            'level': 'DEBUG',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'ncharts_debug.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
        }
    },
    'loggers': {
        'django': {
            'handlers':['django', 'django_debug', 'mail_admins'],
            'propagate': True,
            'level':'DEBUG',
        },
        'datavis': {
            'handlers':['datavis', 'datavis_debug', 'mail_admins'],
            'level':'DEBUG',
        },
        'ncharts': {
            'handlers': ['ncharts_debug', 'ncharts', 'mail_admins'],
            'level': 'DEBUG',
        },
    }
}

if not DEBUG:
    CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
            'LOCATION': 'unix:' + os.path.join(VAR_RUN_DIR, 'django_memcached.sock'),
            # 'LOCATION': '127.0.0.1:11211',
            'TIMEOUT': 300, # 300 seconds is the default
        }
    }
    CACME_MIDDLEWARE_ALIAS = 'default'
    CACHE_MIDDLEWARE_SECONDS = 300
    CACHE_MIDDLEWARE_KEY_PREFIX = ''

SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

INTERNAL_IPS = ['128.117']

<commit_msg>remove FIXME comment-- FIXED; #10<commit_after>"""
Django settings for datavis project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# formerly datavis/settings.py

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os, socket, getpass

BASE_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'abc-123-this-value-is-totally-insecure'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

VAR_RUN_DIR = BASE_DIR
VAR_LIB_DIR = BASE_DIR
LOG_DIR = os.path.join(BASE_DIR, 'log')

# ALLOWED_HOSTS are the server's IP names, NOT the names of allowed client hosts
# (seems like an unfortunate variable name).
# You may see log errors such as:
# Invalid HTTP_HOST header: 'www.baidu.com'. You may need to add 'www.baidu.com' to ALLOWED_HOSTS.
#
# Don't add those external host names to ALLOWED_HOSTS!
# Hacked sites may have a link to this site, but as I understand it, the redirect
# may contain an HTTP packet with an altered HTTP_HOST and SERVER_NAME, hoping that
# a dumb server, thinking HTTP_HOST is itself, will use it in its own redirects and
# <script> statemtents.  The latter could result in an import of hacker code on a
# client's browser. Setting ALLOWED_HOSTS to the various names for datavis will
# result in packets being ignored if they contain other than the following:
ALLOWED_HOSTS = ['datavis', 'datavis.eol.ucar.edu', 'localhost', '128.117.82.210']

# Application definition

# In django, an app is: "a Python package that is specifically intended
# for use in a Django project".
INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # 'django.contrib.formtools',
    'datetimewidget',
    'timezone_field',
    'ncharts',
)

MIDDLEWARE_CLASSES = (
    # https://docs.djangoproject.com/en/1.7/topics/cache/#order-of-middleware-classes
    # UpdateCacheMiddleware must appear before SessionMiddleware,
    # and LocaleMiddleware FetchFromCacheMiddleware must occur after them.
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'datavis.middleware.InternalUseOnlyMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',
)

# TEMPLATE_DEBUG = True
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, 'templates')
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
                # list if you haven't customized them:
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

ROOT_URLCONF = 'datavis.urls'

WSGI_APPLICATION = 'datavis.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(VAR_LIB_DIR, 'db.sqlite3'),
        'OPTIONS': {'timeout': 20,},
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'US/Mountain'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

# URL to use when referring to static files located in STATIC_ROOT
STATIC_URL = '/static/'

# STATIC_ROOT is where "pythyon3 manage.py collectstatic" puts
# the static files it finds.
# See /etc/httpd/conf/vhosts/datavis.conf:
#	Alias /static/ /var/django/eol-django-datavis/static/
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

# People who should receive emails of ERRORs
ADMINS = (
    ('Gordon Maclean', 'maclean@ucar.edu'),
    # ('Hien Nguyen', 'hnguyen@ucar.edu'),
)
EMAIL_HOST = "smtp.eol.ucar.edu"
# Email address they appear to come from
SERVER_EMAIL = getpass.getuser() + '@' + socket.getfqdn()

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format' : "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
            'datefmt' : "%d/%b/%Y %H:%M:%S"
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'django_debug': {
            'level': 'DEBUG',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'django_debug.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'django': {
            'level': 'WARNING',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'django.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'datavis_debug': {
            'level': 'DEBUG',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'datavis_debug.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'datavis': {
            'level': 'WARNING',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'datavis.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'ncharts': {
            'level': 'WARNING',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'ncharts.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'ncharts_debug': {
            'level': 'DEBUG',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(LOG_DIR, 'ncharts_debug.log'),
            'when': 'W6', 'interval': 1, 'backupCount': 10, 'utc': False,
            'formatter': 'verbose'
        },
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
        }
    },
    'loggers': {
        'django': {
            'handlers':['django', 'django_debug', 'mail_admins'],
            'propagate': True,
            'level':'DEBUG',
        },
        'datavis': {
            'handlers':['datavis', 'datavis_debug', 'mail_admins'],
            'level':'DEBUG',
        },
        'ncharts': {
            'handlers': ['ncharts_debug', 'ncharts', 'mail_admins'],
            'level': 'DEBUG',
        },
    }
}

if not DEBUG:
    CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
            'LOCATION': 'unix:' + os.path.join(VAR_RUN_DIR, 'django_memcached.sock'),
            # 'LOCATION': '127.0.0.1:11211',
            'TIMEOUT': 300, # 300 seconds is the default
        }
    }
    CACME_MIDDLEWARE_ALIAS = 'default'
    CACHE_MIDDLEWARE_SECONDS = 300
    CACHE_MIDDLEWARE_KEY_PREFIX = ''

SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

INTERNAL_IPS = ['128.117']

<|endoftext|>


<commit_before>import getopt, sys, os

from . import utils

from . import passes

from . import interface

from . import provenance

from . import pool

from . import driver

def main():
    """This is the main entry point

    razor [--work-dir=<dir>] <manifest>

    """
    return Slash(sys.argv).run() if utils.checkOccamLib() else 1

class Slash(object):

    def __init__(self, argv):
        utils.setLogger()
        (self.flags, self.args) = getopt.getopt(argv[1:], None, ['work-dir='])
        self.manifest = utils.get_manifest(self.args)
        if self.manifest is None:
            self.usage(argv[0])
            self.ok = False
            return
        self.work_dir = utils.get_work_dir(self.flags)
        self.pool = pool.getDefaultPool()
        self.ok = True

    def  usage(self, exe):
        sys.stderr.write('{0} [--work-dir=<dir>] <manifest>\n'.format(exe))

    def run(self):

        if not self.ok: return 1

        if not utils.make_work_dir(self.work_dir): return 1

        (ok, module, binary, libs, native_libs, ldflags, args, name) = utils.check_manifest(self.manifest)

        if not ok: return 1

        sys.stderr.write('\nslash working on {0} wrt {1} ...\n'.format(module, ' '.join(libs)))

        #<delete this once done>
        new_libs = []
        for lib in libs:
            new_libs.append(os.path.realpath(lib))

        libs = new_libs
        #</delete this once done>

        files = utils.populate_work_dir(module, libs, self.work_dir)

        os.chdir(self.work_dir)

        #specialize the arguments ...
        if not (args is None):
            main = files[module]
            pre = main.get()
            post = main.new('a')
            passes.specialize_program_args(pre, post, args, 'arguments', name=name)

        #watches were inserted here ...

        # Internalize everything that we can
        # We can never internalize main
        interface.writeInterface(interface.mainInterface(), 'main.iface')

        # First compute the simple interfaces
        vals = files.items()
        refs = dict([(k, provenance.VersionedFile(utils.prevent_collisions(k[:k.rfind('.bc')]), 'iface'))
                     for (k,_) in vals])

        def _references((m,f)):
            "Computing references"
            name = refs[m].new()
            passes.interface(f.get(), name, [])

        pool.InParallel(_references, vals, self.pool)

        def _internalize((m,i)):
            "Internalizing from references"
            pre = i.get()
            post = i.new('i')
            passes.intern(pre, post,
                         [refs[f].get() for f in refs.keys() if f != m] +
                         ['main.iface'])

        pool.InParallel(_internalize, vals, self.pool)

        # Strip everything
        # This is safe because external names are not stripped
        def _strip(m):
            "Stripping symbols"
            pre = m.get()
            post = m.new('x')
            passes.strip(pre, post)

        if utils.get_flag(self.flags, 'no-strip', None) is None:
            pool.InParallel(_strip, files.values(), self.pool)

        # Begin main loop
        iface_before_file = provenance.VersionedFile('interface_before', 'iface')
        iface_after_file = provenance.VersionedFile('interface_after', 'iface')
        progress = True
        rewrite_files = {}
        for m in files.keys():
            base = utils.prevent_collisions(m[:m.rfind('.bc')])
            rewrite_files[m] = provenance.VersionedFile(base, 'rw')
        iteration = 0
        while progress:
            iteration += 1
            progress = False
            # Intra-module previrt
            def intra(m):
                "Intra-module previrtualization"
                # for m in files.values():
                # intra-module previrt
                pre = m.get()
                pre_base = os.path.basename(pre)
                post = m.new('p')
                post_base = os.path.basename(post)
                fn = 'previrt_%s-%s' % (pre_base, post_base)
                passes.peval(pre, post, log=open(fn, 'w'))
            pool.InParallel(intra, files.values(), self.pool)

            # Inter-module previrt
            iface = passes.deep([x.get() for x in files.values()],
                                ['main.iface'])
            interface.writeInterface(iface, iface_before_file.new())

            # Specialize
            def _spec((nm, m)):
                "Inter-module module specialization"
                pre = m.get()
                post = m.new('s')
                rw = rewrite_files[nm].new()
                passes.specialize(pre, post, rw, [iface_before_file.get()])
            pool.InParallel(_spec, files.items(), self.pool)

            # Rewrite
            def rewrite((nm, m)):
                "Inter-module module rewriting"
                pre = m.get()
                post = m.new('r')
                rws = [rewrite_files[x].get() for x in files.keys()
                       if x != nm]
                out = [None]
                retcode = passes.rewrite(pre, post, rws, output=out)
                fn = 'rewrite_%s-%s' % (os.path.basename(pre),
                                        os.path.basename(post))
                dbg = open(fn, 'w')
                dbg.write(out[0])
                dbg.close()
                return retcode
            rws = pool.InParallel(rewrite, files.items(), self.pool)
            progress = any(rws)

            # Aggressive internalization
            pool.InParallel(_references, vals, self.pool)
            pool.InParallel(_internalize, vals, self.pool)

            # Compute the interface again
            iface = passes.deep([x.get() for x in files.values()],
                               ['main.iface'])
            interface.writeInterface(iface, iface_after_file.new())

            # Prune
            def prune(m):
                "Pruning dead code/variables"
                pre = m.get()
                post = m.new('occam')
                passes.intern(pre, post, [iface_after_file.get()])
            pool.InParallel(prune, files.values(), self.pool)

        # Make symlinks for the "final" versions
        for x in files.values():
            trg = x.base('-final')
            if os.path.exists(trg):
                os.unlink(trg)
            os.symlink(x.get(), trg)

        final_libs = [files[x].get() for x in libs]
        final_module = files[module].get()

        if False:
            print "\nFinal libs: ", final_libs, "\n"
            print "\nFinal module: ", final_module, "\n"

        linker_args = final_libs + native_libs + ldflags

        if False:
            print '\nclang++ {0} -o {1} {2}\n'.format(module, binary, ' '.join(linker_args))

        driver.linker(final_module, binary, linker_args, quiet=False)

        pool.shutdownDefaultPool()

        return 0
<commit_msg>synching..<commit_after>import getopt, sys, os

from . import utils

from . import passes

from . import interface

from . import provenance

from . import pool

from . import driver

def main():
    """This is the main entry point

    razor [--work-dir=<dir>] <manifest>

    """
    return Slash(sys.argv).run() if utils.checkOccamLib() else 1

class Slash(object):

    def __init__(self, argv):
        utils.setLogger()
        (self.flags, self.args) = getopt.getopt(argv[1:], None, ['work-dir='])
        self.manifest = utils.get_manifest(self.args)
        if self.manifest is None:
            self.usage(argv[0])
            self.ok = False
            return
        self.work_dir = utils.get_work_dir(self.flags)
        self.pool = pool.getDefaultPool()
        self.ok = True

    def  usage(self, exe):
        sys.stderr.write('{0} [--work-dir=<dir>] <manifest>\n'.format(exe))

    def run(self):

        if not self.ok: return 1

        if not utils.make_work_dir(self.work_dir): return 1

        (ok, module, binary, libs, native_libs, ldflags, args, name) = utils.check_manifest(self.manifest)

        if not ok: return 1

        no_strip = utils.get_flag(self.flags, 'no-strip', None)

        sys.stderr.write('\nslash working on {0} wrt {1} ...\n'.format(module, ' '.join(libs)))

        #<delete this once done>
        new_libs = []
        for lib in libs:
            new_libs.append(os.path.realpath(lib))

        libs = new_libs
        #</delete this once done>

        files = utils.populate_work_dir(module, libs, self.work_dir)

        os.chdir(self.work_dir)

        #specialize the arguments ...
        if not (args is None):
            main = files[module]
            pre = main.get()
            post = main.new('a')
            passes.specialize_program_args(pre, post, args, 'arguments', name=name)

        #watches were inserted here ...

        # Internalize everything that we can
        # We can never internalize main
        interface.writeInterface(interface.mainInterface(), 'main.iface')

        # First compute the simple interfaces
        vals = files.items()
        refs = dict([(k, provenance.VersionedFile(utils.prevent_collisions(k[:k.rfind('.bc')]), 'iface'))
                     for (k,_) in vals])

        def _references((m,f)):
            "Computing references"
            name = refs[m].new()
            passes.interface(f.get(), name, [])

        pool.InParallel(_references, vals, self.pool)

        def _internalize((m,i)):
            "Internalizing from references"
            pre = i.get()
            post = i.new('i')
            passes.intern(pre, post,
                         [refs[f].get() for f in refs.keys() if f != m] +
                         ['main.iface'])

        pool.InParallel(_internalize, vals, self.pool)

        # Strip everything
        # This is safe because external names are not stripped
        def _strip(m):
            "Stripping symbols"
            pre = m.get()
            post = m.new('x')
            passes.strip(pre, post)

        if utils.get_flag(self.flags, 'no-strip', None) is None:
            pool.InParallel(_strip, files.values(), self.pool)

        # Begin main loop
        iface_before_file = provenance.VersionedFile('interface_before', 'iface')
        iface_after_file = provenance.VersionedFile('interface_after', 'iface')
        progress = True
        rewrite_files = {}
        for m in files.keys():
            base = utils.prevent_collisions(m[:m.rfind('.bc')])
            rewrite_files[m] = provenance.VersionedFile(base, 'rw')
        iteration = 0
        while progress:
            iteration += 1
            progress = False
            # Intra-module previrt
            def intra(m):
                "Intra-module previrtualization"
                # for m in files.values():
                # intra-module previrt
                pre = m.get()
                pre_base = os.path.basename(pre)
                post = m.new('p')
                post_base = os.path.basename(post)
                fn = 'previrt_%s-%s' % (pre_base, post_base)
                passes.peval(pre, post, log=open(fn, 'w'))
            pool.InParallel(intra, files.values(), self.pool)

            # Inter-module previrt
            iface = passes.deep([x.get() for x in files.values()],
                                ['main.iface'])
            interface.writeInterface(iface, iface_before_file.new())

            # Specialize
            def _spec((nm, m)):
                "Inter-module module specialization"
                pre = m.get()
                post = m.new('s')
                rw = rewrite_files[nm].new()
                passes.specialize(pre, post, rw, [iface_before_file.get()])
            pool.InParallel(_spec, files.items(), self.pool)

            # Rewrite
            def rewrite((nm, m)):
                "Inter-module module rewriting"
                pre = m.get()
                post = m.new('r')
                rws = [rewrite_files[x].get() for x in files.keys()
                       if x != nm]
                out = [None]
                retcode = passes.rewrite(pre, post, rws, output=out)
                fn = 'rewrite_%s-%s' % (os.path.basename(pre),
                                        os.path.basename(post))
                dbg = open(fn, 'w')
                dbg.write(out[0])
                dbg.close()
                return retcode
            rws = pool.InParallel(rewrite, files.items(), self.pool)
            progress = any(rws)

            # Aggressive internalization
            pool.InParallel(_references, vals, self.pool)
            pool.InParallel(_internalize, vals, self.pool)

            # Compute the interface again
            iface = passes.deep([x.get() for x in files.values()],
                               ['main.iface'])
            interface.writeInterface(iface, iface_after_file.new())

            # Prune
            def prune(m):
                "Pruning dead code/variables"
                pre = m.get()
                post = m.new('occam')
                passes.intern(pre, post, [iface_after_file.get()])
            pool.InParallel(prune, files.values(), self.pool)

        # Make symlinks for the "final" versions
        for x in files.values():
            trg = x.base('-final')
            if os.path.exists(trg):
                os.unlink(trg)
            os.symlink(x.get(), trg)

        final_libs = [files[x].get() for x in libs]
        final_module = files[module].get()

        if False:
            print "\nFinal libs: ", final_libs, "\n"
            print "\nFinal module: ", final_module, "\n"

        linker_args = final_libs + native_libs + ldflags

        if False:
            print '\nclang++ {0} -o {1} {2}\n'.format(module, binary, ' '.join(linker_args))

        driver.linker(final_module, binary, linker_args, quiet=False)

        pool.shutdownDefaultPool()

        return 0
<|endoftext|>