<commit_before>'''Term class hierarchy.'''

# pylint: disable-msg=W0142

from aterm import types
from aterm import exceptions
from aterm import _helpers

class Term(object):
	'''Base class for all terms.'''

	def __init__(self, factory, annotations = None):
		self.factory = factory
		self.__annotations = annotations

	def getFactory(self):
		'''Retrieves the factory responsible for creating this Term.'''
		return self.factory

	def getType(self):
		'''Gets the type of this term.'''
		raise NotImplementedError

	def getHash(self):
		'''Generate a hash value for this term.'''
		return _helpers.Hash()(self)

	def __hash__(self):
		'''Shorthand for getHash().'''
		return self.getHash()

	def isConstant(self):
		'''Whether this term is types, as opposed to have variables or wildcards.'''
		return _helpers.isConstant(self)

	def isEquivalent(self, other):
		'''Checks for structural equivalence of this term agains another term.'''
		return _helpers.isEquivalent(self, other)

	def isEqual(self, other):
		'''Checks equality of this term against another term.  Note that for two
		terms to be equal, any annotations they might have must be equal as
		well.'''
		return _helpers.isEqual(self, other)

	def __eq__(self, other):
		'''Shorthand for the isEqual method.'''
		return self.isEqual(other)

	def match(self, other, args = None, kargs = None):
		'''Matches this term agains a string or term pattern.'''

		if isinstance(other, basestring):
			other = self.factory.parse(other)

		compare = _helpers.PatternComparator(args, kargs)
		return compare(self, other)

	def getAnnotation(self, label):
		'''Gets an annotation associated with label'''
		if not self.__annotations is None:
			annotations = self.__annotations
			while not annotations.isEmpty():
				if label.isEquivalent(annotations.getHead()):
					return annotations.getTail().getHead()
				annotations = annotations.getTail().getTail()
		raise ValueError("undefined annotation '%r'" % label)

	def setAnnotation(self, label, annotation):
		'''Returns a new version of this term with the
		annotation associated with this label added or updated.'''
		return self.setAnnotations(self._setAnnotation(label, annotation, self.getAnnotations()))

	def _setAnnotation(self, label, annotation, annotations):
		if annotations.isEmpty():
			return self.factory.makeCons(label, self.factory.makeCons(annotation, annotations))

		_label = annotations.getHead()
		annotations = annotations.getTail()
		_annotation = annotations.getHead()
		annotations = annotations.getTail()

		if label.isEquivalent(_label):
			return self.factory.makeCons(label, self.factory.makeCons(annotation, annotations))
		else:
			return self.factory.makeCons(_label, self.factory.makeCons(_annotation, self._setAnnotation(label, annotation, annotations)))

	def removeAnnotation(self, label):
		'''Returns a new version of this term with the
		annotation associated with this label removed.'''
		return self.setAnnotations(self._removeAnnotation(label, self.getAnnotations()))

	def _removeAnnotation(self, label, annotations):
		if annotations.isEmpty():
			return annotations

		_label = annotations.getHead()
		annotations = annotations.getTail()
		_annotation = annotations.getHead()
		annotations = annotations.getTail()

		if label.isEquivalent(_label):
			return annotations
		else:
			return self.factory.makeCons(_label, self.factory.makeCons(_annotation, self._removeAnnotation(label, annotations)))

	def getAnnotations(self):
		'''Returns the annotation list.'''
		if self.__annotations is None:
			return self.factory.makeNil()
		else:
			return self.__annotations

	def setAnnotations(self, annotations):
		raise NotImplementedError

	def __getattr__(self, name):
		'''Provide attributes 'type' and 'annotations', shorthand  for
		getType() and getAnnotations() methods respectively.'''
		if name == 'type':
			return self.getType()
		elif name == 'annotations':
			return self.getAnnotations()
		else:
			raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name))

	def __setattr__(self, name, value):
		'''Prevent modification of term attributes'''

		if name in self.__dict__ or name in ('type', 'annotations'):
			raise TypeError("attempt to modify read-only term attribute '%s'" % name)
		else:
			self.__dict__[name] = value

	def make(self, *args, **kargs):
		'''Create a new term based on this term and a list of arguments.'''
		return self._make(args, kargs)

	def _make(self, args, kargs):
		return self

	def accept(self, visitor, *args, **kargs):
		'''Accept a visitor.'''
		raise NotImplementedError

	def writeToTextFile(self, fp):
		'''Write this term to a file object.'''
		writer = _helpers.TextWriter(fp)
		writer.visit(self)

	def __str__(self):
		'''Get the string representation of this term.'''
		try:
			from cStringIO import StringIO
		except ImportError:
			from StringIO import StringIO
		fp = StringIO()
		self.writeToTextFile(fp)
		return fp.getvalue()

	def __repr__(self):
		return str(self)

class Literal(Term):
	'''Base class for literal terms.'''

	def __init__(self, factory, value, annotations = None):
		Term.__init__(self, factory, annotations)
		self.value = value

	def getValue(self):
		return self.value

class Integer(Literal):
	'''Integer literal term.'''

	def getType(self):
		return types.INT

	def __int__(self):
		return int(self.value)

	def setAnnotations(self, annotations):
		return self.factory.makeInt(self.value, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitInt(self, *args, **kargs)

class Real(Literal):
	'''Real literal term.'''

	def getType(self):
		return types.REAL

	def __float__(self):
		return float(self.value)

	def setAnnotations(self, annotations):
		return self.factory.makeReal(self.value, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitReal(self, *args, **kargs)

class String(Literal):
	'''String literal term.'''

	def getType(self):
		return types.STR

	def setAnnotations(self, annotations):
		return self.factory.makeStr(self.value, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitStr(self, *args, **kargs)

class List(Term):
	'''Base class for list terms.'''

	def getType(self):
		return types.LIST

	def isEmpty(self):
		raise NotImplementedError

	def getLength(self):
		raise NotImplementedError

	def __len__(self):
		return self.getLength()

	def getHead(self):
		raise NotImplementedError

	def getTail(self):
		raise NotImplementedError

	def __getitem__(self, index):
		if self.isEmpty():
			raise IndexError
		elif index == 0:
			return self.getHead()
		else:
			return self.getTail().__getitem__(index - 1)

	# TODO: write an __iter__ method

	def insert(self, element):
		return self.factory.makeCons(element, self)

	def append(self, element):
		return self.extend(self.factory.makeConst(element, self.factory.makeNil()))

	def extend(self, element):
		raise NotImplementedError

	def accept(self, visitor, *args, **kargs):
		return visitor.visitList(self, *args, **kargs)

class Nil(List):
	'''Empty list term.'''

	def __init__(self, factory, annotations = None):
		List.__init__(self, factory, annotations)

	def isEmpty(self):
		return True

	def getLength(self):
		return 0

	def getHead(self):
		raise exceptions.EmptyListException

	def getTail(self):
		raise exceptions.EmptyListException

	def extend(self, tail):
		return tail

	def setAnnotations(self, annotations):
		return self.factory.makeNil(annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitNil(self, *args, **kargs)

class Cons(List):
	'''Concatenated list term.'''

	def __init__(self, factory, head, tail = None, annotations = None):
		List.__init__(self, factory, annotations)

		if not isinstance(head, Term):
			raise TypeError("head is not a term: %r" % head)
		self.head = head
		if tail is None:
			self.tail = self.factory.makeNil()
		else:
			if not isinstance(tail, (List, Variable, Wildcard)):
				raise TypeError("tail is not a list, variable, or wildcard term: %r" % tail)
			self.tail = tail

	def isEmpty(self):
		return False

	def getLength(self):
		return 1 + self.tail.getLength()

	def getHead(self):
		return self.head

	def getTail(self):
		return self.tail

	def _make(self, args, kargs):
		return self.factory.makeCons(
			self.head._make(args, kargs),
			self.tail._make(args, kargs),
			self.annotations
		)

	def extend(self, tail):
		return self.factory.makeCons(
			self.head,
			self.tail.append(tail),
			self.getAnnotations()
		)

	def setAnnotations(self, annotations):
		return self.factory.makeCons(self.head, self.tail, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitCons(self, *args, **kargs)

class Application(Term):
	'''Application term.'''

	def __init__(self, factory, name, args = None, annotations = None):
		Term.__init__(self, factory, annotations)

		if not isinstance(name, (String, Variable, Wildcard)):
			raise TypeError("name is not a string, variable, or wildcard term: %r" % name)
		self.name = name
		if args is None:
			self.args = self.factory.makeNil()
		else:
			if not isinstance(args, (List, Variable, Wildcard)):
				raise TypeError("args is not a list term: %r" % args)
			self.args = args

	def getType(self):
		return types.APPL

	def getName(self):
		return self.name

	def getArity(self):
		return self.args.getLength()

	def getArgs(self):
		return self.args

	def _make(self, args, kargs):
		return self.factory.makeAppl(self.name._make(args, kargs), self.args._make(args, kargs), self.annotations)

	def setAnnotations(self, annotations):
		return self.factory.makeAppl(self.name, self.args, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitAppl(self, *args, **kargs)

class Placeholder(Term):
	'''Base class for placeholder terms.'''
	pass

class Wildcard(Placeholder):
	'''Wildcard term.'''

	def getType(self):
		return types.WILDCARD

	def _make(self, args, kargs):
		try:
			return args.pop(0)
		except IndexError:
			raise TypeError('insufficient number of arguments')

	def setAnnotations(self, annotations):
		return self.factory.makeWildcard(annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitWildcard(self, *args, **kargs)

class Variable(Placeholder):
	'''Variable term.'''

	def __init__(self, factory, name, pattern, annotations = None):
		Placeholder.__init__(self, factory, annotations)
		self.name = name
		self.pattern = pattern

	def getType(self):
		return types.VAR

	def getName(self):
		return self.name

	def getPattern(self):
		return self.pattern

	def _make(self, args, kargs):
		name = self.getName()
		if name in kargs:
			# TODO: do something with the pattern here?
			return kargs[name]
		else:
			raise ValueError('undefined term variable %s' % name)

	def setAnnotations(self, annotations):
		return self.factory.makeVar(self.name, self.pattern, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitVar(self, *args, **kargs)

<commit_msg>Use slots in terms.<commit_after>'''Term class hierarchy.'''

# pylint: disable-msg=W0142

from aterm import types
from aterm import exceptions
from aterm import _helpers

class Term(object):
	'''Base class for all terms.'''

	__slots__ = ['factory', '__annotations']

	def __init__(self, factory, annotations = None):
		self.factory = factory
		self.__annotations = annotations

	def getFactory(self):
		'''Retrieves the factory responsible for creating this Term.'''
		return self.factory

	def getType(self):
		'''Gets the type of this term.'''
		raise NotImplementedError

	def getHash(self):
		'''Generate a hash value for this term.'''
		return _helpers.Hash()(self)

	def __hash__(self):
		'''Shorthand for getHash().'''
		return self.getHash()

	def isConstant(self):
		'''Whether this term is types, as opposed to have variables or wildcards.'''
		return _helpers.isConstant(self)

	def isEquivalent(self, other):
		'''Checks for structural equivalence of this term agains another term.'''
		return _helpers.isEquivalent(self, other)

	def isEqual(self, other):
		'''Checks equality of this term against another term.  Note that for two
		terms to be equal, any annotations they might have must be equal as
		well.'''
		return _helpers.isEqual(self, other)

	def __eq__(self, other):
		'''Shorthand for the isEqual method.'''
		return self.isEqual(other)

	def match(self, other, args = None, kargs = None):
		'''Matches this term agains a string or term pattern.'''

		if isinstance(other, basestring):
			other = self.factory.parse(other)

		compare = _helpers.PatternComparator(args, kargs)
		return compare(self, other)

	def getAnnotation(self, label):
		'''Gets an annotation associated with label'''
		if not self.__annotations is None:
			annotations = self.__annotations
			while not annotations.isEmpty():
				if label.isEquivalent(annotations.getHead()):
					return annotations.getTail().getHead()
				annotations = annotations.getTail().getTail()
		raise ValueError("undefined annotation '%r'" % label)

	def setAnnotation(self, label, annotation):
		'''Returns a new version of this term with the
		annotation associated with this label added or updated.'''
		return self.setAnnotations(self._setAnnotation(label, annotation, self.getAnnotations()))

	def _setAnnotation(self, label, annotation, annotations):
		if annotations.isEmpty():
			return self.factory.makeCons(label, self.factory.makeCons(annotation, annotations))

		_label = annotations.getHead()
		annotations = annotations.getTail()
		_annotation = annotations.getHead()
		annotations = annotations.getTail()

		if label.isEquivalent(_label):
			return self.factory.makeCons(label, self.factory.makeCons(annotation, annotations))
		else:
			return self.factory.makeCons(_label, self.factory.makeCons(_annotation, self._setAnnotation(label, annotation, annotations)))

	def removeAnnotation(self, label):
		'''Returns a new version of this term with the
		annotation associated with this label removed.'''
		return self.setAnnotations(self._removeAnnotation(label, self.getAnnotations()))

	def _removeAnnotation(self, label, annotations):
		if annotations.isEmpty():
			return annotations

		_label = annotations.getHead()
		annotations = annotations.getTail()
		_annotation = annotations.getHead()
		annotations = annotations.getTail()

		if label.isEquivalent(_label):
			return annotations
		else:
			return self.factory.makeCons(_label, self.factory.makeCons(_annotation, self._removeAnnotation(label, annotations)))

	def getAnnotations(self):
		'''Returns the annotation list.'''
		if self.__annotations is None:
			return self.factory.makeNil()
		else:
			return self.__annotations

	def setAnnotations(self, annotations):
		raise NotImplementedError

	def __getattr__(self, name):
		'''Provide attributes 'type' and 'annotations', shorthand  for
		getType() and getAnnotations() methods respectively.'''
		if name == 'type':
			return self.getType()
		elif name == 'annotations':
			return self.getAnnotations()
		else:
			raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name))

	def __setattr__(self, name, value):
		'''Prevent modification of term attributes'''

		# TODO: implement this with a metaclass

		try:
			object.__getattribute__(self, name)
		except AttributeError:
			object.__setattr__(self, name, value)
		else:
			raise TypeError("attempt to modify read-only term attribute '%s'" % name)

	def make(self, *args, **kargs):
		'''Create a new term based on this term and a list of arguments.'''
		return self._make(args, kargs)

	def _make(self, args, kargs):
		return self

	def accept(self, visitor, *args, **kargs):
		'''Accept a visitor.'''
		raise NotImplementedError

	def writeToTextFile(self, fp):
		'''Write this term to a file object.'''
		writer = _helpers.TextWriter(fp)
		writer.visit(self)

	def __str__(self):
		'''Get the string representation of this term.'''
		try:
			from cStringIO import StringIO
		except ImportError:
			from StringIO import StringIO
		fp = StringIO()
		self.writeToTextFile(fp)
		return fp.getvalue()

	def __repr__(self):
		return str(self)

class Literal(Term):
	'''Base class for literal terms.'''

	__slots__ = ['value']

	def __init__(self, factory, value, annotations = None):
		Term.__init__(self, factory, annotations)
		self.value = value

	def getValue(self):
		return self.value

class Integer(Literal):
	'''Integer literal term.'''

	__slots__ = []

	def getType(self):
		return types.INT

	def __int__(self):
		return int(self.value)

	def setAnnotations(self, annotations):
		return self.factory.makeInt(self.value, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitInt(self, *args, **kargs)

class Real(Literal):
	'''Real literal term.'''

	__slots__ = []

	def getType(self):
		return types.REAL

	def __float__(self):
		return float(self.value)

	def setAnnotations(self, annotations):
		return self.factory.makeReal(self.value, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitReal(self, *args, **kargs)

class String(Literal):
	'''String literal term.'''

	__slots__ = []

	def getType(self):
		return types.STR

	def setAnnotations(self, annotations):
		return self.factory.makeStr(self.value, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitStr(self, *args, **kargs)

class List(Term):
	'''Base class for list terms.'''

	__slots__ = []

	def getType(self):
		return types.LIST

	def isEmpty(self):
		raise NotImplementedError

	def getLength(self):
		raise NotImplementedError

	def __len__(self):
		return self.getLength()

	def getHead(self):
		raise NotImplementedError

	def getTail(self):
		raise NotImplementedError

	def __getitem__(self, index):
		if self.isEmpty():
			raise IndexError
		elif index == 0:
			return self.getHead()
		else:
			return self.getTail().__getitem__(index - 1)

	# TODO: write an __iter__ method

	def insert(self, element):
		return self.factory.makeCons(element, self)

	def append(self, element):
		return self.extend(self.factory.makeConst(element, self.factory.makeNil()))

	def extend(self, element):
		raise NotImplementedError

	def accept(self, visitor, *args, **kargs):
		return visitor.visitList(self, *args, **kargs)

class Nil(List):
	'''Empty list term.'''

	__slots__ = []

	def __init__(self, factory, annotations = None):
		List.__init__(self, factory, annotations)

	def isEmpty(self):
		return True

	def getLength(self):
		return 0

	def getHead(self):
		raise exceptions.EmptyListException

	def getTail(self):
		raise exceptions.EmptyListException

	def extend(self, tail):
		return tail

	def setAnnotations(self, annotations):
		return self.factory.makeNil(annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitNil(self, *args, **kargs)

class Cons(List):
	'''Concatenated list term.'''

	__slots__ = ['head', 'tail']

	def __init__(self, factory, head, tail = None, annotations = None):
		List.__init__(self, factory, annotations)

		if not isinstance(head, Term):
			raise TypeError("head is not a term: %r" % head)
		self.head = head
		if tail is None:
			self.tail = self.factory.makeNil()
		else:
			if not isinstance(tail, (List, Variable, Wildcard)):
				raise TypeError("tail is not a list, variable, or wildcard term: %r" % tail)
			self.tail = tail

	def isEmpty(self):
		return False

	def getLength(self):
		return 1 + self.tail.getLength()

	def getHead(self):
		return self.head

	def getTail(self):
		return self.tail

	def _make(self, args, kargs):
		return self.factory.makeCons(
			self.head._make(args, kargs),
			self.tail._make(args, kargs),
			self.annotations
		)

	def extend(self, tail):
		return self.factory.makeCons(
			self.head,
			self.tail.append(tail),
			self.getAnnotations()
		)

	def setAnnotations(self, annotations):
		return self.factory.makeCons(self.head, self.tail, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitCons(self, *args, **kargs)

class Application(Term):
	'''Application term.'''

	__slots__ = ['name', 'args']

	def __init__(self, factory, name, args = None, annotations = None):
		Term.__init__(self, factory, annotations)

		if not isinstance(name, (String, Variable, Wildcard)):
			raise TypeError("name is not a string, variable, or wildcard term: %r" % name)
		self.name = name
		if args is None:
			self.args = self.factory.makeNil()
		else:
			if not isinstance(args, (List, Variable, Wildcard)):
				raise TypeError("args is not a list term: %r" % args)
			self.args = args

	def getType(self):
		return types.APPL

	def getName(self):
		return self.name

	def getArity(self):
		return self.args.getLength()

	def getArgs(self):
		return self.args

	def _make(self, args, kargs):
		return self.factory.makeAppl(self.name._make(args, kargs), self.args._make(args, kargs), self.annotations)

	def setAnnotations(self, annotations):
		return self.factory.makeAppl(self.name, self.args, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitAppl(self, *args, **kargs)

class Placeholder(Term):
	'''Base class for placeholder terms.'''

	__slots__ = []

class Wildcard(Placeholder):
	'''Wildcard term.'''

	__slots__ = []

	def getType(self):
		return types.WILDCARD

	def _make(self, args, kargs):
		try:
			return args.pop(0)
		except IndexError:
			raise TypeError('insufficient number of arguments')

	def setAnnotations(self, annotations):
		return self.factory.makeWildcard(annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitWildcard(self, *args, **kargs)

class Variable(Placeholder):
	'''Variable term.'''

	__slots__ = ['name', 'pattern']

	def __init__(self, factory, name, pattern, annotations = None):
		Placeholder.__init__(self, factory, annotations)
		self.name = name
		self.pattern = pattern

	def getType(self):
		return types.VAR

	def getName(self):
		return self.name

	def getPattern(self):
		return self.pattern

	def _make(self, args, kargs):
		name = self.getName()
		if name in kargs:
			# TODO: do something with the pattern here?
			return kargs[name]
		else:
			raise ValueError('undefined term variable %s' % name)

	def setAnnotations(self, annotations):
		return self.factory.makeVar(self.name, self.pattern, annotations)

	def accept(self, visitor, *args, **kargs):
		return visitor.visitVar(self, *args, **kargs)

<|endoftext|>


<commit_before>from __future__ import with_statement

import imp
import inspect
import os
import sys

from attest         import ast, statistics
from attest.codegen import to_source, SourceGenerator

__all__ = ['COMPILES_AST',
           'ExpressionEvaluator',
           'TestFailure',
           'assert_hook',
           'AssertTransformer',
           'AssertImportHook',
          ]

try:
    compile(ast.parse('pass'), '<string>', 'exec')
except TypeError:
    COMPILES_AST = False
else:
    COMPILES_AST = True

class ExpressionEvaluator(SourceGenerator):
    """Evaluates `expr` in the context of `globals` and `locals`, expanding
    the values of variables and the results of binary operations, but
    keeping comparison and boolean operators.

    .. testsetup::

        from attest import ExpressionEvaluator

    >>> var = 1 + 2
    >>> value = ExpressionEvaluator('var == 5 - 3', globals(), locals())
    >>> repr(value)
    '(3 == 2)'
    >>> bool(value)
    False

    .. versionadded:: 0.5

    """

    def __init__(self, expr, globals, locals):
        self.expr = expr
        self.globals, self.locals = globals, locals
        self.result = []
        self.node = ast.parse(self.expr).body[0].value

    # Trigger visit after init because we don't want to
    # evaluate twice in case of a successful assert
    def late_visit(self):
        self.visit(self.node)

    def __repr__(self):
        return ''.join(self.result)

    def __str__(self):
        return '\n'.join((self.expr, repr(self)))

    def __nonzero__(self):
        return bool(eval(self.expr, self.globals, self.locals))

    def eval(self, node):
        return eval(to_source(node), self.globals, self.locals)

    def write(self, s):
        self.result.append(str(s))

    def visit_Name(self, node):
        value = self.eval(node)
        if getattr(value, '__name__', None):
            self.write(value.__name__)
        else:
            self.write(repr(value))

    def generic_visit(self, node):
        self.write(repr(self.eval(node)))

    visit_BinOp = visit_Subscript = generic_visit
    visit_ListComp = visit_GeneratorExp = generic_visit
    visit_SetComp = visit_DictComp = generic_visit
    visit_Call = visit_Attribute = generic_visit

class TestFailure(AssertionError):
    """Extended :exc:`AssertionError` used by the assert hook.

    :param value: The asserted expression evaluated with
        :class:`ExpressionEvaluator`.
    :param msg: Optional message passed to the assertion.

    .. versionadded:: 0.5

    """

    def __init__(self, value, msg=''):
        self.value = value
        AssertionError.__init__(self, msg)

def assert_hook(expr, msg='', globals=None, locals=None):
    """Like `assert`, but using :class:`ExpressionEvaluator`. If
    you import this in test modules and the :class:`AssertImportHook` is
    installed (which it is automatically the first time you import from
    :mod:`attest`), `assert` statements are rewritten as a call to
    this.

    The import must be a top-level *from* import, example::

        from attest import Tests, assert_hook

    .. versionadded:: 0.5

    """
    statistics.assertions += 1
    if globals is None:
        globals = inspect.stack()[1][0].f_globals
    if locals is None:
        locals = inspect.stack()[1][0].f_locals
    value = ExpressionEvaluator(expr, globals, locals)
    if not value:
        # Visit only if assertion fails
        value.late_visit()
        raise TestFailure(value, msg)

# Build AST nodes on 2.5 more easily
def _build(node, **kwargs):
    node = node()
    for key, value in kwargs.iteritems():
        setattr(node, key, value)
    return node

class AssertTransformer(ast.NodeTransformer):
    """Parses `source` with :mod:`_ast` and transforms `assert`
    statements into calls to :func:`assert_hook`.

    .. warning::

        CPython 2.5 doesn't compile AST nodes and when that fails this
        transformer will generate source code from the AST instead. While
        Attest's own tests passes on CPython 2.5, there might be code that
        it currently would render back incorrectly, most likely resulting
        in a failure. Because Python's syntax is simple, this isn't very
        likely, but you might want to :meth:`~AssertImportHook.disable` the
        import hook if you test regularly on CPython 2.5.

        It also messes up the line numbers so they don't match the original
        source code, meaning tracebacks will point to the line numbers in
        the *generated* source and preview the code on that line in the
        *original* source. The improved error message with the import hook
        is often worth it however, and failures will still point to the
        right file and function.

    .. versionadded:: 0.5

    """

    def __init__(self, source, filename=''):
        self.source = source
        self.filename = filename

    @property
    def should_rewrite(self):
        """:const:`True` if the source imports :func:`assert_hook`."""
        return ('assert_hook' in self.source and
                any(s.module == 'attest' and
                    any(n.name == 'assert_hook' for n in s.names)
                    for s in ast.parse(self.source).body
                    if isinstance(s, ast.ImportFrom)))

    def make_module(self, name, newpath=None):
        """Compiles the transformed code into a module object which it also
        inserts in :data:`sys.modules`.

        :returns: The module object.

        """
        module = imp.new_module(name)
        module.__file__ = self.filename
        if newpath:
            module.__path__ = newpath
        exec self.code in vars(module)
        sys.modules[name] = module
        return module

    @property
    def node(self):
        """The transformed AST node."""
        node = ast.parse(self.source, self.filename)
        node = self.visit(node)
        ast.fix_missing_locations(node)
        return node

    @property
    def code(self):
        """The :attr:`node` compiled into a code object."""
        if COMPILES_AST:
            return compile(self.node, self.filename, 'exec')
        return compile(to_source(self.node), self.filename, 'exec')

    def visit_Assert(self, node):
        args = [_build(ast.Str, s=to_source(node.test)),
                node.msg if node.msg is not None else _build(ast.Str, s=''),
                _build(ast.Call,
                    func=_build(ast.Name, id='globals', ctx=ast.Load()),
                    args=[], keywords=[], starargs=None, kwargs=None),
                _build(ast.Call,
                    func=_build(ast.Name, id='locals', ctx=ast.Load()),
                    args=[], keywords=[], starargs=None, kwargs=None)
               ]
        return ast.copy_location(
            _build(ast.Expr, value=_build(ast.Call,
                   func=_build(ast.Name, id='assert_hook', ctx=ast.Load()),
                   args=args, keywords=[], starargs=None, kwargs=None)), node)

class AssertImportHookEnabledDescriptor(object):

    def __get__(self, instance, owner):
        return any(isinstance(ih, owner) for ih in sys.meta_path)

class AssertImportHook(object):
    """An :term:`importer` that transforms imported modules with
    :class:`AssertTransformer`.

    .. versionadded:: 0.5

    """

    #: Class property, :const:`True` if the hook is enabled.
    enabled = AssertImportHookEnabledDescriptor()

    @classmethod
    def enable(cls):
        """Enable the import hook."""
        cls.disable()
        sys.meta_path.insert(0, cls())

    @classmethod
    def disable(cls):
        """Disable the import hook."""
        sys.meta_path[:] = [ih for ih in sys.meta_path
                               if not isinstance(ih, cls)]

    def __init__(self):
        self._cache = {}

    def __enter__(self):
        sys.meta_path.insert(0, self)

    def __exit__(self, exc_type, exc_value, traceback):
        sys.meta_path.remove(self)

    def find_module(self, name, path=None):
        lastname = name.rsplit('.', 1)[-1]
        try:
            self._cache[name] = imp.find_module(lastname, path), path
        except ImportError:
            return
        return self

    def load_module(self, name):
        if name in sys.modules:
            return sys.modules[name]

        source, filename, newpath = self.get_source(name)
        (fd, fn, info), path = self._cache[name]

        if source is None:
            return imp.load_module(name, fd, fn, info)

        transformer = AssertTransformer(source, filename)

        if not transformer.should_rewrite:
            fd, fn, info = imp.find_module(name.rsplit('.', 1)[-1], path)
            return imp.load_module(name, fd, fn, info)

        try:
            return transformer.make_module(name, newpath)
        except Exception, err:
            raise ImportError('cannot import %s: %s' % (name, err))

    def get_source(self, name):
        try:
            (fd, fn, info), path = self._cache[name]
        except KeyError:
            raise ImportError(name)

        code = filename = newpath = None
        if info[2] == imp.PY_SOURCE:
            filename = fn
            with fd:
                code = fd.read()
        elif info[2] == imp.PY_COMPILED:
            filename = fn[:-1]
            with open(filename, 'U') as f:
                code = f.read()
        elif info[2] == imp.PKG_DIRECTORY:
            filename = os.path.join(fn, '__init__.py')
            newpath = [fn]
            with open(filename, 'U') as f:
                code = f.read()

        return code, filename, newpath
<commit_msg>Fix undefined var in closure<commit_after>from __future__ import with_statement

import imp
import inspect
import os
import sys

from attest         import ast, statistics
from attest.codegen import to_source, SourceGenerator

__all__ = ['COMPILES_AST',
           'ExpressionEvaluator',
           'TestFailure',
           'assert_hook',
           'AssertTransformer',
           'AssertImportHook',
          ]

try:
    compile(ast.parse('pass'), '<string>', 'exec')
except TypeError:
    COMPILES_AST = False
else:
    COMPILES_AST = True

class ExpressionEvaluator(SourceGenerator):
    """Evaluates `expr` in the context of `globals` and `locals`, expanding
    the values of variables and the results of binary operations, but
    keeping comparison and boolean operators.

    .. testsetup::

        from attest import ExpressionEvaluator

    >>> var = 1 + 2
    >>> value = ExpressionEvaluator('var == 5 - 3', globals(), locals())
    >>> repr(value)
    '(3 == 2)'
    >>> bool(value)
    False

    .. versionadded:: 0.5

    """

    def __init__(self, expr, globals, locals):
        self.expr = expr
        # Putting locals in globals for closures
        self.globals = dict(globals)
        self.locals = locals
        self.globals.update(self.locals)

        self.result = []
        self.node = ast.parse(self.expr).body[0].value

    # Trigger visit after init because we don't want to
    # evaluate twice in case of a successful assert
    def late_visit(self):
        self.visit(self.node)

    def __repr__(self):
        return ''.join(self.result)

    def __str__(self):
        return '\n'.join((self.expr, repr(self)))

    def __nonzero__(self):
        return bool(eval(self.expr, self.globals, self.locals))

    def eval(self, node):
        return eval(to_source(node), self.globals, self.locals)

    def write(self, s):
        self.result.append(str(s))

    def visit_Name(self, node):
        value = self.eval(node)
        if getattr(value, '__name__', None):
            self.write(value.__name__)
        else:
            self.write(repr(value))

    def generic_visit(self, node):
        self.write(repr(self.eval(node)))

    visit_BinOp = visit_Subscript = generic_visit
    visit_ListComp = visit_GeneratorExp = generic_visit
    visit_SetComp = visit_DictComp = generic_visit
    visit_Call = visit_Attribute = generic_visit

class TestFailure(AssertionError):
    """Extended :exc:`AssertionError` used by the assert hook.

    :param value: The asserted expression evaluated with
        :class:`ExpressionEvaluator`.
    :param msg: Optional message passed to the assertion.

    .. versionadded:: 0.5

    """

    def __init__(self, value, msg=''):
        self.value = value
        AssertionError.__init__(self, msg)

def assert_hook(expr, msg='', globals=None, locals=None):
    """Like `assert`, but using :class:`ExpressionEvaluator`. If
    you import this in test modules and the :class:`AssertImportHook` is
    installed (which it is automatically the first time you import from
    :mod:`attest`), `assert` statements are rewritten as a call to
    this.

    The import must be a top-level *from* import, example::

        from attest import Tests, assert_hook

    .. versionadded:: 0.5

    """
    statistics.assertions += 1
    if globals is None:
        globals = inspect.stack()[1][0].f_globals
    if locals is None:
        locals = inspect.stack()[1][0].f_locals
    value = ExpressionEvaluator(expr, globals, locals)
    if not value:
        # Visit only if assertion fails
        value.late_visit()
        raise TestFailure(value, msg)

# Build AST nodes on 2.5 more easily
def _build(node, **kwargs):
    node = node()
    for key, value in kwargs.iteritems():
        setattr(node, key, value)
    return node

class AssertTransformer(ast.NodeTransformer):
    """Parses `source` with :mod:`_ast` and transforms `assert`
    statements into calls to :func:`assert_hook`.

    .. warning::

        CPython 2.5 doesn't compile AST nodes and when that fails this
        transformer will generate source code from the AST instead. While
        Attest's own tests passes on CPython 2.5, there might be code that
        it currently would render back incorrectly, most likely resulting
        in a failure. Because Python's syntax is simple, this isn't very
        likely, but you might want to :meth:`~AssertImportHook.disable` the
        import hook if you test regularly on CPython 2.5.

        It also messes up the line numbers so they don't match the original
        source code, meaning tracebacks will point to the line numbers in
        the *generated* source and preview the code on that line in the
        *original* source. The improved error message with the import hook
        is often worth it however, and failures will still point to the
        right file and function.

    .. versionadded:: 0.5

    """

    def __init__(self, source, filename=''):
        self.source = source
        self.filename = filename

    @property
    def should_rewrite(self):
        """:const:`True` if the source imports :func:`assert_hook`."""
        return ('assert_hook' in self.source and
                any(s.module == 'attest' and
                    any(n.name == 'assert_hook' for n in s.names)
                    for s in ast.parse(self.source).body
                    if isinstance(s, ast.ImportFrom)))

    def make_module(self, name, newpath=None):
        """Compiles the transformed code into a module object which it also
        inserts in :data:`sys.modules`.

        :returns: The module object.

        """
        module = imp.new_module(name)
        module.__file__ = self.filename
        if newpath:
            module.__path__ = newpath
        exec self.code in vars(module)
        sys.modules[name] = module
        return module

    @property
    def node(self):
        """The transformed AST node."""
        node = ast.parse(self.source, self.filename)
        node = self.visit(node)
        ast.fix_missing_locations(node)
        return node

    @property
    def code(self):
        """The :attr:`node` compiled into a code object."""
        if COMPILES_AST:
            return compile(self.node, self.filename, 'exec')
        return compile(to_source(self.node), self.filename, 'exec')

    def visit_Assert(self, node):
        args = [_build(ast.Str, s=to_source(node.test)),
                node.msg if node.msg is not None else _build(ast.Str, s=''),
                _build(ast.Call,
                    func=_build(ast.Name, id='globals', ctx=ast.Load()),
                    args=[], keywords=[], starargs=None, kwargs=None),
                _build(ast.Call,
                    func=_build(ast.Name, id='locals', ctx=ast.Load()),
                    args=[], keywords=[], starargs=None, kwargs=None)
               ]
        return ast.copy_location(
            _build(ast.Expr, value=_build(ast.Call,
                   func=_build(ast.Name, id='assert_hook', ctx=ast.Load()),
                   args=args, keywords=[], starargs=None, kwargs=None)), node)

class AssertImportHookEnabledDescriptor(object):

    def __get__(self, instance, owner):
        return any(isinstance(ih, owner) for ih in sys.meta_path)

class AssertImportHook(object):
    """An :term:`importer` that transforms imported modules with
    :class:`AssertTransformer`.

    .. versionadded:: 0.5

    """

    #: Class property, :const:`True` if the hook is enabled.
    enabled = AssertImportHookEnabledDescriptor()

    @classmethod
    def enable(cls):
        """Enable the import hook."""
        cls.disable()
        sys.meta_path.insert(0, cls())

    @classmethod
    def disable(cls):
        """Disable the import hook."""
        sys.meta_path[:] = [ih for ih in sys.meta_path
                               if not isinstance(ih, cls)]

    def __init__(self):
        self._cache = {}

    def __enter__(self):
        sys.meta_path.insert(0, self)

    def __exit__(self, exc_type, exc_value, traceback):
        sys.meta_path.remove(self)

    def find_module(self, name, path=None):
        lastname = name.rsplit('.', 1)[-1]
        try:
            self._cache[name] = imp.find_module(lastname, path), path
        except ImportError:
            return
        return self

    def load_module(self, name):
        if name in sys.modules:
            return sys.modules[name]

        source, filename, newpath = self.get_source(name)
        (fd, fn, info), path = self._cache[name]

        if source is None:
            return imp.load_module(name, fd, fn, info)

        transformer = AssertTransformer(source, filename)

        if not transformer.should_rewrite:
            fd, fn, info = imp.find_module(name.rsplit('.', 1)[-1], path)
            return imp.load_module(name, fd, fn, info)

        try:
            return transformer.make_module(name, newpath)
        except Exception, err:
            raise ImportError('cannot import %s: %s' % (name, err))

    def get_source(self, name):
        try:
            (fd, fn, info), path = self._cache[name]
        except KeyError:
            raise ImportError(name)

        code = filename = newpath = None
        if info[2] == imp.PY_SOURCE:
            filename = fn
            with fd:
                code = fd.read()
        elif info[2] == imp.PY_COMPILED:
            filename = fn[:-1]
            with open(filename, 'U') as f:
                code = f.read()
        elif info[2] == imp.PKG_DIRECTORY:
            filename = os.path.join(fn, '__init__.py')
            newpath = [fn]
            with open(filename, 'U') as f:
                code = f.read()

        return code, filename, newpath
<|endoftext|>


<commit_before># Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

import os

from manager_rest.deployment_update.constants import STATES

from integration_tests import AgentlessTestCase
from integration_tests.tests.utils import get_resource as resource

BLUEPRINT_ID = 'update-blueprint'
blueprints_base_path = 'dsl/deployment_update'

class TimeoutException(Exception):
    pass

class DeploymentUpdateBase(AgentlessTestCase):
    def _wait_for_update(self, dep_update):
        self.wait_for_execution_to_end(
            self.client.executions.get(dep_update.execution_id))
        self.assertEqual(
            self.client.deployment_updates.get(dep_update.id).state,
            STATES.SUCCESSFUL
        )

    def _assert_relationship(self,
                             relationships,
                             target,
                             expected_type=None,
                             exists=True):
        """
        assert that a node/node instance has a specific relationship
        :param relationships: node/node instance relationships list
        :param target: target name (node id, not instance id)
        :param expected_type: expected relationship type
        """
        expected_type = expected_type or 'cloudify.relationships.contained_in'
        error_msg = 'relationship of target "{0}" ' \
                    'and type "{1}" is missing'.format(target, expected_type)

        for relationship in relationships:
            relationship_type = relationship['type']
            relationship_target = (relationship.get('target_name') or
                                   relationship.get('target_id'))

            if (relationship_type == expected_type and
                    relationship_target == target):
                if not exists:
                    self.fail(error_msg)
                return

        if exists:
            self.fail(error_msg.format(target, expected_type))

    def _deploy_and_get_modified_bp_path(self,
                                         test_name,
                                         inputs=None,
                                         deployment_id=None):
        base_bp_path, modified_bp_path = \
            self._get_base_and_modified_bp_path(test_name)

        deployment, _ = self.deploy_application(
                base_bp_path,
                inputs=inputs,
                deployment_id=deployment_id)

        return deployment, modified_bp_path

    def _get_base_and_modified_bp_path(self, test_name):
        base_dir = os.path.join(test_name, 'base')
        modified_dir = os.path.join(test_name, 'modification')
        base_bp = '{0}_base.yaml'.format(test_name)
        modified_bp = '{0}_modification.yaml'.format(test_name)

        base_bp_path = self._get_blueprint_path(base_dir, base_bp)
        modified_bp_path = self._get_blueprint_path(modified_dir, modified_bp)
        return base_bp_path, modified_bp_path

    def _get_blueprint_path(self, blueprint_folder, blueprint_file):
        return resource(
                os.path.join(blueprints_base_path,
                             blueprint_folder,
                             blueprint_file))

    def _map_node_and_node_instances(self, deployment_id, dct):
        nodes_dct = {k: [] for k, _ in dct.items()}
        node_instances_dct = {k: [] for k, _ in dct.items()}

        for k, v in dct.items():
            nodes = list(self.client.nodes.list(
                deployment_id=deployment_id,
                node_id=dct[k]
            ))
            node_instances = \
                list(self.client.node_instances.list(
                    deployment_id=deployment_id,
                    node_id=dct[k]
                ))
            nodes_dct[k].extend(nodes)
            node_instances_dct[k].extend(node_instances)

        return nodes_dct, node_instances_dct

    def _assert_equal_entity_dicts(self,
                                   old_nodes,
                                   new_nodes,
                                   keys,
                                   excluded_items=None):
        old_nodes_by_id = \
            {n['id']: n for k in keys for n in old_nodes.get(k, {})}
        new_nodes_by_id = \
            {n['id']: n for k in keys for n in new_nodes.get(k, {})}

        intersecting_ids = \
            set(old_nodes_by_id.keys()) & set(new_nodes_by_id.keys())
        for id in intersecting_ids:
            self._assert_equal_dicts(old_nodes_by_id[id],
                                     new_nodes_by_id[id],
                                     excluded_items=excluded_items)

    def _assert_equal_dicts(self, d1, d2, excluded_items=None):
        # the blueprint id will always change in deployment update
        excluded_items = excluded_items or []
        excluded_items.append('blueprint_id')
        for k, v in d1.items():
            if k not in excluded_items:
                # Assuming here that only node instances have a version
                # in their dict, and so `version_changed` is only applicable
                # in the cases where we're comparing node instances
                if k == 'version':
                    self.assertGreaterEqual(d2.get(k, None), v,
                                            'New version should be greater '
                                            ' or equal to the old version. '
                                            'Old: {0}, new: {1}'
                                            .format(d1[k], d2[k]))
                else:
                    self.assertEqual(d2.get(k, None), v,
                                     '{0} has changed on {1}. {2}!={3}'
                                     .format(d1, k, d1[k], d2[k]))

    def _create_dict(self, path):
        if len(path) == 1:
            return path[0]
        else:
            return {path[0]: self._create_dict(path[1:])}

    @staticmethod
    def _assertDictContainsSubset(subset, containing_dict):
        subset.items() <= containing_dict.items()
<commit_msg>dep-update inte-tests: s/node_id/id/ (#3193)<commit_after># Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

import os

from manager_rest.deployment_update.constants import STATES

from integration_tests import AgentlessTestCase
from integration_tests.tests.utils import get_resource as resource

BLUEPRINT_ID = 'update-blueprint'
blueprints_base_path = 'dsl/deployment_update'

class TimeoutException(Exception):
    pass

class DeploymentUpdateBase(AgentlessTestCase):
    def _wait_for_update(self, dep_update):
        self.wait_for_execution_to_end(
            self.client.executions.get(dep_update.execution_id))
        self.assertEqual(
            self.client.deployment_updates.get(dep_update.id).state,
            STATES.SUCCESSFUL
        )

    def _assert_relationship(self,
                             relationships,
                             target,
                             expected_type=None,
                             exists=True):
        """
        assert that a node/node instance has a specific relationship
        :param relationships: node/node instance relationships list
        :param target: target name (node id, not instance id)
        :param expected_type: expected relationship type
        """
        expected_type = expected_type or 'cloudify.relationships.contained_in'
        error_msg = 'relationship of target "{0}" ' \
                    'and type "{1}" is missing'.format(target, expected_type)

        for relationship in relationships:
            relationship_type = relationship['type']
            relationship_target = (relationship.get('target_name') or
                                   relationship.get('target_id'))

            if (relationship_type == expected_type and
                    relationship_target == target):
                if not exists:
                    self.fail(error_msg)
                return

        if exists:
            self.fail(error_msg.format(target, expected_type))

    def _deploy_and_get_modified_bp_path(self,
                                         test_name,
                                         inputs=None,
                                         deployment_id=None):
        base_bp_path, modified_bp_path = \
            self._get_base_and_modified_bp_path(test_name)

        deployment, _ = self.deploy_application(
                base_bp_path,
                inputs=inputs,
                deployment_id=deployment_id)

        return deployment, modified_bp_path

    def _get_base_and_modified_bp_path(self, test_name):
        base_dir = os.path.join(test_name, 'base')
        modified_dir = os.path.join(test_name, 'modification')
        base_bp = '{0}_base.yaml'.format(test_name)
        modified_bp = '{0}_modification.yaml'.format(test_name)

        base_bp_path = self._get_blueprint_path(base_dir, base_bp)
        modified_bp_path = self._get_blueprint_path(modified_dir, modified_bp)
        return base_bp_path, modified_bp_path

    def _get_blueprint_path(self, blueprint_folder, blueprint_file):
        return resource(
                os.path.join(blueprints_base_path,
                             blueprint_folder,
                             blueprint_file))

    def _map_node_and_node_instances(self, deployment_id, dct):
        nodes_dct = {k: [] for k, _ in dct.items()}
        node_instances_dct = {k: [] for k, _ in dct.items()}

        for k, v in dct.items():
            nodes = list(self.client.nodes.list(
                deployment_id=deployment_id,
                id=dct[k]
            ))
            node_instances = \
                list(self.client.node_instances.list(
                    deployment_id=deployment_id,
                    node_id=dct[k]
                ))
            nodes_dct[k].extend(nodes)
            node_instances_dct[k].extend(node_instances)

        return nodes_dct, node_instances_dct

    def _assert_equal_entity_dicts(self,
                                   old_nodes,
                                   new_nodes,
                                   keys,
                                   excluded_items=None):
        old_nodes_by_id = \
            {n['id']: n for k in keys for n in old_nodes.get(k, {})}
        new_nodes_by_id = \
            {n['id']: n for k in keys for n in new_nodes.get(k, {})}

        intersecting_ids = \
            set(old_nodes_by_id.keys()) & set(new_nodes_by_id.keys())
        for id in intersecting_ids:
            self._assert_equal_dicts(old_nodes_by_id[id],
                                     new_nodes_by_id[id],
                                     excluded_items=excluded_items)

    def _assert_equal_dicts(self, d1, d2, excluded_items=None):
        # the blueprint id will always change in deployment update
        excluded_items = excluded_items or []
        excluded_items.append('blueprint_id')
        for k, v in d1.items():
            if k not in excluded_items:
                # Assuming here that only node instances have a version
                # in their dict, and so `version_changed` is only applicable
                # in the cases where we're comparing node instances
                if k == 'version':
                    self.assertGreaterEqual(d2.get(k, None), v,
                                            'New version should be greater '
                                            ' or equal to the old version. '
                                            'Old: {0}, new: {1}'
                                            .format(d1[k], d2[k]))
                else:
                    self.assertEqual(d2.get(k, None), v,
                                     '{0} has changed on {1}. {2}!={3}'
                                     .format(d1, k, d1[k], d2[k]))

    def _create_dict(self, path):
        if len(path) == 1:
            return path[0]
        else:
            return {path[0]: self._create_dict(path[1:])}

    @staticmethod
    def _assertDictContainsSubset(subset, containing_dict):
        subset.items() <= containing_dict.items()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

import logging
import re
import scrapy

from scrapy import Request
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
from scrapy.utils.url import url_has_any_extension

from typing import Any, Generator, List, Optional, Tuple

class BaseDocumentationSpider(scrapy.Spider):
    name = None  # type: Optional[str]
    # Exclude domain address.
    deny_domains = []  # type: List[str]
    start_urls = []  # type: List[str]
    deny = []  # type: List[str]
    file_extensions = ['.' + ext for ext in IGNORED_EXTENSIONS]  # type: List[str]
    tags = ('a', 'area', 'img')
    attrs = ('href', 'src')

    def __init__(self, *args, **kwargs):
        # type: (*Any, **Any) -> None
        super(BaseDocumentationSpider, self).__init__(*args, **kwargs)
        self.has_error = False

    def _set_error_state(self):
        # type: () -> None
        self.has_error = True

    def _has_extension(self, url):
        # type: (str) -> bool
        return url_has_any_extension(url, self.file_extensions)

    def _is_external_url(self, url):
        # type: (str) -> bool
        return url.startswith('http') or self._has_extension(url)

    def check_existing(self, response):
        # type: (Any) -> None
        self.log(response)

    def check_permalink(self, response):
        # type: (Any) -> None
        self.log(response)
        xpath_template = "//*[@id='{permalink}' or @name='{permalink}']"
        m = re.match(r".+\#(?P<permalink>.*)$", response.request.url)  # Get anchor value.
        if not m:
            return
        permalink = m.group('permalink')
        # Check permalink existing on response page.
        if not response.selector.xpath(xpath_template.format(permalink=permalink)):
            self._set_error_state()
            raise Exception(
                "Permalink #{} is not found on page {}".format(permalink, response.request.url))

    def parse(self, response):
        # type: (Any) -> Generator[Request, None, None]
        self.log(response)
        for link in LxmlLinkExtractor(deny_domains=self.deny_domains, deny_extensions=['doc'],
                                      tags=self.tags, attrs=self.attrs, deny=self.deny,
                                      canonicalize=False).extract_links(response):
            callback = self.parse  # type: Any
            dont_filter = False
            method = 'GET'
            if self._is_external_url(link.url):
                callback = self.check_existing
                method = 'HEAD'
            elif '#' in link.url:
                dont_filter = True
                callback = self.check_permalink
            yield Request(link.url, method=method, callback=callback, dont_filter=dont_filter,
                          errback=self.error_callback)

    def retry_request_with_get(self, request):
        # type: (Request) -> Generator[Request, None, None]
        request.method = 'GET'
        request.dont_filter = True
        yield request

    def error_callback(self, failure):
        # type: (Any) -> Optional[Generator[Any, None, None]]
        if hasattr(failure.value, 'response') and failure.value.response:
            response = failure.value.response
            if response.status == 404:
                self._set_error_state()
                raise Exception('Page not found: {}'.format(response))
            if response.status == 405 and response.request.method == 'HEAD':
                # Method 'HEAD' not allowed, repeat request with 'GET'
                return self.retry_request_with_get(response.request)
            self.log("Error! Please check link: {}".format(response), logging.ERROR)
        elif isinstance(failure.type, IOError):
            self._set_error_state()
        else:
            raise Exception(failure.value)
        return None
<commit_msg>documentation_crawler: Add exclude list.<commit_after>#!/usr/bin/env python
from __future__ import print_function

import logging
import re
import scrapy

from scrapy import Request
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
from scrapy.utils.url import url_has_any_extension

from typing import Any, Generator, List, Optional, Tuple

EXCLUDED_URLS = [
    # Google calendar returns 404s on HEAD requests unconditionally
    'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com',
    # Returns 409 errors to HEAD requests frequently
    'https://medium.freecodecamp.com',
]

class BaseDocumentationSpider(scrapy.Spider):
    name = None  # type: Optional[str]
    # Exclude domain address.
    deny_domains = []  # type: List[str]
    start_urls = []  # type: List[str]
    deny = []  # type: List[str]
    file_extensions = ['.' + ext for ext in IGNORED_EXTENSIONS]  # type: List[str]
    tags = ('a', 'area', 'img')
    attrs = ('href', 'src')

    def __init__(self, *args, **kwargs):
        # type: (*Any, **Any) -> None
        super(BaseDocumentationSpider, self).__init__(*args, **kwargs)
        self.has_error = False

    def _set_error_state(self):
        # type: () -> None
        self.has_error = True

    def _has_extension(self, url):
        # type: (str) -> bool
        return url_has_any_extension(url, self.file_extensions)

    def _is_external_url(self, url):
        # type: (str) -> bool
        return url.startswith('http') or self._has_extension(url)

    def check_existing(self, response):
        # type: (Any) -> None
        self.log(response)

    def check_permalink(self, response):
        # type: (Any) -> None
        self.log(response)
        xpath_template = "//*[@id='{permalink}' or @name='{permalink}']"
        m = re.match(r".+\#(?P<permalink>.*)$", response.request.url)  # Get anchor value.
        if not m:
            return
        permalink = m.group('permalink')
        # Check permalink existing on response page.
        if not response.selector.xpath(xpath_template.format(permalink=permalink)):
            self._set_error_state()
            raise Exception(
                "Permalink #{} is not found on page {}".format(permalink, response.request.url))

    def parse(self, response):
        # type: (Any) -> Generator[Request, None, None]
        self.log(response)
        for link in LxmlLinkExtractor(deny_domains=self.deny_domains, deny_extensions=['doc'],
                                      tags=self.tags, attrs=self.attrs, deny=self.deny,
                                      canonicalize=False).extract_links(response):
            callback = self.parse  # type: Any
            dont_filter = False
            method = 'GET'
            if self._is_external_url(link.url):
                callback = self.check_existing
                method = 'HEAD'
            elif '#' in link.url:
                dont_filter = True
                callback = self.check_permalink
            yield Request(link.url, method=method, callback=callback, dont_filter=dont_filter,
                          errback=self.error_callback)

    def retry_request_with_get(self, request):
        # type: (Request) -> Generator[Request, None, None]
        request.method = 'GET'
        request.dont_filter = True
        yield request

    def exclude_error(self, url):
        if url in EXCLUDED_URLS:
            return True
        return False

    def error_callback(self, failure):
        # type: (Any) -> Optional[Generator[Any, None, None]]
        if hasattr(failure.value, 'response') and failure.value.response:
            response = failure.value.response
            if self.exclude_error(response.url):
                return None
            if response.status == 404:
                self._set_error_state()
                raise Exception('Page not found: {}'.format(response))
            if response.status == 405 and response.request.method == 'HEAD':
                # Method 'HEAD' not allowed, repeat request with 'GET'
                return self.retry_request_with_get(response.request)
            self.log("Error! Please check link: {}".format(response), logging.ERROR)
        elif isinstance(failure.type, IOError):
            self._set_error_state()
        else:
            raise Exception(failure.value)
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python

# author: brendan@shellshockcomputer.com.au

import ConfigParser
from bottle import route, install, run, template, static_file, response, PasteServer, post, request#, debug
from bottle_sqlite import SQLitePlugin
import json
import urllib
import urllib2
import datetime

config = ConfigParser.RawConfigParser()
config.read('config.ini')

install(SQLitePlugin(dbfile=(config.get("pool", "database"))))

@route('/api/btime')
def blocktime():
    response.headers['Cache-Control'] = 'public, max-age=100'
    payload = {
        'requestType': 'getForging',
        'secretPhrase': config.get("pool", "poolphrase")
    }
    opener = urllib2.build_opener(urllib2.HTTPHandler())
    data = urllib.urlencode(payload)
    forging = json.loads(opener.open(config.get("pool", "nhzhost")+'/nhz', data=data).read())
    getdl = forging["deadline"]
    dl = str(datetime.timedelta(seconds=getdl))
    return {'blocktime': dl}

@route('/api/accounts')
def apiaccounts(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    getlastheight = db.execute("SELECT height FROM blocks ORDER BY timestamp DESC").fetchone()

    try:
        lastheight = getlastheight[0]
    except:
        lastheight = 0

    c = db.execute("SELECT ars, heightfrom, heightto, CAST(amount AS FLOAT)/100000000 AS amount FROM leased WHERE heightto > %s" % (lastheight)).fetchall()
    accounts = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return accounts

@route('/api/blocks')
def apiblocks(db):
    response.headers['Cache-Control'] = 'public, max-age=1800'
    c = db.execute("SELECT height, timestamp, CAST(totalfee AS FLOAT)/100000000 AS totalfee FROM blocks WHERE totalfee > 0 ORDER BY timestamp DESC").fetchall()
    blocks = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return blocks

@route('/api/leased')
def apileased():
    getaccounts = json.loads(urllib2.urlopen(config.get("pool", "nhzhost")+"/nhz?requestType=getAccount&account="+config.get("pool", "poolaccount")).read())

    try:
        leasebal = getaccounts['effectiveBalanceNHZ']
    except:
        leasebal = 0

    return {'blocktime': leasebal}

@route('/api/payouts')
def apipayouts(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    c = db.execute("SELECT account, CAST(fee AS FLOAT)/100000000 AS fee, payment FROM payouts DESC").fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/paid')
def apipaid(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    c = db.execute("SELECT blocktime, account, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid>0 ORDER BY blocktime DESC").fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/unpaid')
def apiunpaid(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    c = db.execute("SELECT blocktime, account, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid=0 ORDER BY blocktime DESC").fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/userunpaid/:user#[0-9]+#')
def apiuserunpaid(db, user):
    response.headers['Cache-Control'] = 'public, max-age=600'
    c = db.execute("SELECT blocktime, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid=0 and account LIKE ?", (user,)).fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/userpaid/:user#[0-9]+#')
def apiuserpaid(db, user):
    response.headers['Cache-Control'] = 'public, max-age=600'
    c = db.execute("SELECT blocktime, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid>0 and account LIKE ?", (user,)).fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/')
def default(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    poolaccount = config.get("pool", "poolaccountrs")
    poolfee = config.get("pool", "feePercent")
    db.text_factory = str
    d = db.execute("SELECT height, timestamp, CAST(totalfee AS FLOAT)/100000000 AS totalfee FROM blocks ORDER BY timestamp DESC limit 1")
    getlastheight = d.fetchone()

    try:
        lastheight = getlastheight[0]
    except:
        lastheight = 0

    c = db.execute("SELECT ars, heightto, CAST(amount AS FLOAT)/100000000 AS amount FROM leased ORDER BY heightfrom DESC limit 6")
    result = c.fetchall()
    e = db.execute("SELECT height, timestamp, CAST(totalfee AS FLOAT)/100000000 AS totalfee FROM blocks ORDER BY timestamp DESC limit 6")
    block = e.fetchall()
    getaccounts = json.loads(urllib2.urlopen(config.get("pool", "nhzhost")+"/nhz?requestType=getAccount&account="+config.get("pool", "poolaccount")).read())
    try:
        leasebal = getaccounts['effectiveBalanceNHZ']
    except KeyError:
        leasebal = 0

    output = template('default', pa=poolaccount, fee=poolfee, rows=result, blocks=block, nhzb=leasebal)
    return output

@route('/static/:path#.+#', name='static')
def static(path):
    response.headers['Cache-Control'] = 'public, max-age=2592000'
    return static_file(path, root='static')

@route('/favicon.ico')
def get_favicon():
    response.headers['Cache-Control'] = 'public, max-age=2592000'
    return static('favicon.ico')

@route('/getting_started')
def getting_started():
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('gettingstarted')
    return output

@route('/accounts')
def accounts(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('accounts')
    return output

@route('/blocks')
def blocks(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('blocks')
    return output

@route('/payouts')
def payouts(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('payouts')
    return output

@route('/unpaid')
def unpaid(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    cunpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid=0").fetchone()
    for e in cunpaid:
        try:
            unpaid = float(e)/100000000
        except TypeError:
            unpaid = 0

    output = template('unpaid', unpaid=unpaid)
    return output

@route('/paid')
def paid(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    cpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid>0").fetchone()
    for d in cpaid:
        try:
            paid = float(d)/100000000
        except TypeError:
            paid = 0

    output = template('paid', paid=paid)
    return output

@post('/user')
def user(db):
    aid = request.forms.get('username')
    if aid.isdigit():
        user = aid
        c = db.execute("SELECT ars FROM leased WHERE account LIKE ?", (aid,)).fetchone()
        for ids in c:
            aid = ids

    else:
        c = db.execute("SELECT account FROM leased WHERE ars LIKE ?", (aid,)).fetchone()
        for users in c:
            user = users

    cpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid>0 and account LIKE ?", (user,)).fetchone()
    for d in cpaid:
        try:
            paid = float(d)/100000000
        except TypeError:
            paid = 0

    cunpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid=0 and account LIKE ?", (user,)).fetchone()
    for e in cunpaid:
        try:
            unpaid = float(e)/100000000
        except TypeError:
            unpaid = 0

    output = template('user', user=user, aid=aid, paid=paid, unpaid=unpaid)
    return output

#debug(True)
run(server=PasteServer, port=8810, host='0.0.0.0')<commit_msg>remove lasthieght<commit_after>#!/usr/bin/env python

# author: brendan@shellshockcomputer.com.au

import ConfigParser
from bottle import route, install, run, template, static_file, response, PasteServer, post, request#, debug
from bottle_sqlite import SQLitePlugin
import json
import urllib
import urllib2
import datetime

config = ConfigParser.RawConfigParser()
config.read('config.ini')

install(SQLitePlugin(dbfile=(config.get("pool", "database"))))

@route('/api/btime')
def blocktime():
    response.headers['Cache-Control'] = 'public, max-age=100'
    payload = {
        'requestType': 'getForging',
        'secretPhrase': config.get("pool", "poolphrase")
    }
    opener = urllib2.build_opener(urllib2.HTTPHandler())
    data = urllib.urlencode(payload)
    forging = json.loads(opener.open(config.get("pool", "nhzhost")+'/nhz', data=data).read())
    getdl = forging["deadline"]
    dl = str(datetime.timedelta(seconds=getdl))
    return {'blocktime': dl}

@route('/api/accounts')
def apiaccounts(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    getlastheight = db.execute("SELECT height FROM blocks ORDER BY timestamp DESC").fetchone()

    try:
        lastheight = getlastheight[0]
    except:
        lastheight = 0

    c = db.execute("SELECT ars, heightfrom, heightto, CAST(amount AS FLOAT)/100000000 AS amount FROM leased WHERE heightto > %s" % (lastheight)).fetchall()
    accounts = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return accounts

@route('/api/blocks')
def apiblocks(db):
    response.headers['Cache-Control'] = 'public, max-age=1800'
    c = db.execute("SELECT height, timestamp, CAST(totalfee AS FLOAT)/100000000 AS totalfee FROM blocks WHERE totalfee > 0 ORDER BY timestamp DESC").fetchall()
    blocks = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return blocks

@route('/api/leased')
def apileased():
    getaccounts = json.loads(urllib2.urlopen(config.get("pool", "nhzhost")+"/nhz?requestType=getAccount&account="+config.get("pool", "poolaccount")).read())

    try:
        leasebal = getaccounts['effectiveBalanceNHZ']
    except:
        leasebal = 0

    return {'blocktime': leasebal}

@route('/api/payouts')
def apipayouts(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    c = db.execute("SELECT account, CAST(fee AS FLOAT)/100000000 AS fee, payment FROM payouts DESC").fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/paid')
def apipaid(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    c = db.execute("SELECT blocktime, account, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid>0 ORDER BY blocktime DESC").fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/unpaid')
def apiunpaid(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    c = db.execute("SELECT blocktime, account, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid=0 ORDER BY blocktime DESC").fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/userunpaid/:user#[0-9]+#')
def apiuserunpaid(db, user):
    response.headers['Cache-Control'] = 'public, max-age=600'
    c = db.execute("SELECT blocktime, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid=0 and account LIKE ?", (user,)).fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/api/userpaid/:user#[0-9]+#')
def apiuserpaid(db, user):
    response.headers['Cache-Control'] = 'public, max-age=600'
    c = db.execute("SELECT blocktime, percentage, CAST(amount AS FLOAT)/100000000 AS amount FROM accounts WHERE paid>0 and account LIKE ?", (user,)).fetchall()
    pays = json.dumps( [dict(ix) for ix in c], separators=(',',':'))
    return pays

@route('/')
def default(db):
    response.headers['Cache-Control'] = 'public, max-age=3600'
    poolaccount = config.get("pool", "poolaccountrs")
    poolfee = config.get("pool", "feePercent")
    c = db.execute("SELECT ars, heightto, CAST(amount AS FLOAT)/100000000 AS amount FROM leased ORDER BY heightfrom DESC limit 6")
    result = c.fetchall()
    e = db.execute("SELECT height, timestamp, CAST(totalfee AS FLOAT)/100000000 AS totalfee FROM blocks ORDER BY timestamp DESC limit 6")
    block = e.fetchall()
    getaccounts = json.loads(urllib2.urlopen(config.get("pool", "nhzhost")+"/nhz?requestType=getAccount&account="+config.get("pool", "poolaccount")).read())
    try:
        leasebal = getaccounts['effectiveBalanceNHZ']
    except KeyError:
        leasebal = 0

    output = template('default', pa=poolaccount, fee=poolfee, rows=result, blocks=block, nhzb=leasebal)
    return output

@route('/static/:path#.+#', name='static')
def static(path):
    response.headers['Cache-Control'] = 'public, max-age=2592000'
    return static_file(path, root='static')

@route('/favicon.ico')
def get_favicon():
    response.headers['Cache-Control'] = 'public, max-age=2592000'
    return static('favicon.ico')

@route('/getting_started')
def getting_started():
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('gettingstarted')
    return output

@route('/accounts')
def accounts(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('accounts')
    return output

@route('/blocks')
def blocks(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('blocks')
    return output

@route('/payouts')
def payouts(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    output = template('payouts')
    return output

@route('/unpaid')
def unpaid(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    cunpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid=0").fetchone()
    for e in cunpaid:
        try:
            unpaid = float(e)/100000000
        except TypeError:
            unpaid = 0

    output = template('unpaid', unpaid=unpaid)
    return output

@route('/paid')
def paid(db):
    response.headers['Cache-Control'] = 'public, max-age=43200'
    cpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid>0").fetchone()
    for d in cpaid:
        try:
            paid = float(d)/100000000
        except TypeError:
            paid = 0

    output = template('paid', paid=paid)
    return output

@post('/user')
def user(db):
    aid = request.forms.get('username')
    if aid.isdigit():
        user = aid
        c = db.execute("SELECT ars FROM leased WHERE account LIKE ?", (aid,)).fetchone()
        for ids in c:
            aid = ids

    else:
        c = db.execute("SELECT account FROM leased WHERE ars LIKE ?", (aid,)).fetchone()
        for users in c:
            user = users

    cpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid>0 and account LIKE ?", (user,)).fetchone()
    for d in cpaid:
        try:
            paid = float(d)/100000000
        except TypeError:
            paid = 0

    cunpaid = db.execute("SELECT sum(amount) FROM accounts WHERE paid=0 and account LIKE ?", (user,)).fetchone()
    for e in cunpaid:
        try:
            unpaid = float(e)/100000000
        except TypeError:
            unpaid = 0

    output = template('user', user=user, aid=aid, paid=paid, unpaid=unpaid)
    return output

#debug(True)
run(server=PasteServer, port=8810, host='0.0.0.0')<|endoftext|>


<commit_before>#!/usr/bin/python3

import os
import sys
import time
import threading
import subprocess

from bottle import run, request, static_file, route, redirect, parse_date, HTTPResponse

location = os.path.dirname(__file__)
static_files = os.path.join(location, 'static')

class Picture:
    refresh_after_seconds = 5

    def __init__(self):
        self.content = self.get_content()
        self.time = time.time()
        self.next_refresh_time = self.time + self.refresh_after_seconds

    @staticmethod
    def get_content():
        return subprocess.check_output(
            "raspistill -o - -e jpg -ex night -w 640 -h 480 -t 100".split(),
            stderr = sys.stderr)

    def is_outdated(self):
        return self.next_refresh_time < time.time()

capture_lock = threading.Lock()
capturing = False
picture = Picture()

@route('/')
def home():
    return redirect('/static/index.html')

@route('/static/<filename:path>')
def static(filename):
    return static_file(filename, root = static_files)

def renew_picture():
    global capturing, picture
    with capture_lock:
        to_capture = not capturing and picture.is_outdated()
        if to_capture:
            capturing = True # critical section
    if to_capture:
        picture = Picture()
        capturing = False

@route('/picture.jpg')
def serve_picture():
    renew_picture()
    _picture = picture

    # from bottle.static_file
    headers = dict()
    headers['Content-Type'] = 'image/jpeg'

    headers['Content-Length'] = clen = len(_picture.content)
    lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(picture.time))
    headers['Last-Modified'] = lm

    ims = request.environ.get('HTTP_IF_MODIFIED_SINCE')
    if ims:
        ims = parse_date(ims.split(";")[0].strip())
    if ims is not None and ims >= int(picture.time):
        headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
        return HTTPResponse(status=304, **headers)

    body = '' if request.method == 'HEAD' else picture.content
    return HTTPResponse(body, **headers)

if __name__ == "__main__":
    # server option: http://bottlepy.org/docs/dev/deployment.html#server-options
    run(host = '0.0.0.0', port = 80, debug = True, server='cherrypy')
<commit_msg>removed race-condition<commit_after>#!/usr/bin/python3

import os
import sys
import time
import threading
import subprocess

from bottle import run, request, static_file, route, redirect, parse_date, HTTPResponse

location = os.path.dirname(__file__)
static_files = os.path.join(location, 'static')

class Picture:
    refresh_after_seconds = 5

    def __init__(self):
        self.content = self.get_content()
        self.time = time.time()
        self.next_refresh_time = self.time + self.refresh_after_seconds

    @staticmethod
    def get_content():
        return subprocess.check_output(
            "raspistill -o - -e jpg -ex night -w 640 -h 480 -t 100".split(),
            stderr = sys.stderr)

    def is_outdated(self):
        return self.next_refresh_time < time.time()

capture_lock = threading.Lock()
capturing = False
picture = Picture()

@route('/')
def home():
    return redirect('/static/index.html')

@route('/static/<filename:path>')
def static(filename):
    return static_file(filename, root = static_files)

def renew_picture():
    global capturing, picture
    with capture_lock:
        to_capture = not capturing and picture.is_outdated()
        if to_capture:
            capturing = True # critical section
    if to_capture:
        picture = Picture()
        capturing = False

@route('/picture.jpg')
def serve_picture():
    renew_picture()
    _picture = picture

    # from bottle.static_file
    headers = dict()
    headers['Content-Type'] = 'image/jpeg'

    headers['Content-Length'] = clen = len(_picture.content)
    lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(_picture.time))
    headers['Last-Modified'] = lm

    ims = request.get_header('If-Modified-Since')
    if not ims:
      ims = request.environ.get('HTTP_IF_MODIFIED_SINCE')
    if ims:
        ims = parse_date(ims.split(";")[0].strip())
    if ims is not None and ims >= int(_picture.time):
        headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
        return HTTPResponse(status=304, **headers)

    body = '' if request.method == 'HEAD' else _picture.content
    return HTTPResponse(body, **headers)

if __name__ == "__main__":
    # server option: http://bottlepy.org/docs/dev/deployment.html#server-options
    run(host = '0.0.0.0', port = 80, debug = True, server='cherrypy')
<|endoftext|>