<commit_before>import abc
import logging
import inspect
from urllib.parse import urlparse, parse_qsl
from itertools import zip_longest
from collections import OrderedDict
from functools import partial

from .util import cached_property

log = logging.getLogger(__name__)
sentinel = object()

_PAGE_METHOD = object()
_RESOURCE_METHOD = object()
_RESOURCE = object()
_FORM_CTYPE = b'application/x-www-form-urlencoded'

class LegacyMultiDict(object):
    """Utilitary class which wrap dict to make it suitable for old utilities
    like wtforms"""

    def __init__(self, dic):
        self._dic = dic

    def getlist(self, k):
        return [self._dic[k]]

    def __contains__(self, k):
        return k in self._dic

    def __iter__(self):
        for k in self._dic:
            yield k

class Request(object):
    __slots__ = ('__dict__',
        # the excpected property names and names of respective zerogw items
        'uri',           # !Uri
        'content_type',  # !Header Content-Type
        'cookie',        # !Header Cookie
        'body',          # !PostBody
        )

    @cached_property
    def parsed_uri(self):
        return urlparse(self.uri.decode('ascii'))

    @cached_property
    def form_arguments(self):
        arguments = {}
        if hasattr(self, 'uri'):
            arguments.update(parse_qsl(self.parsed_uri.query))
        body = getattr(self, 'body', None)
        if body and getattr(self, 'content_type', None) == _FORM_CTYPE:
            arguments.update(parse_qsl(self.body.decode('ascii')))
        return arguments

    @cached_property
    def legacy_arguments(self):
        return LegacyMultiDict(self.form_arguments)

    @cached_property
    def cookies(self):
        cobj = SimpleCookie(self.cookie.decode('ascii', 'ignore'))
        return dict((k, cobj[k].value) for k in cobj)

class WebException(Exception):
    """Base for all exceptions which render error code (and page) to client"""

    @abc.abstractmethod
    def default_response(self):
        pass

class Forbidden(WebException):

    def default_response(self):
        return (b'403 Forbidden',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>403 Forbidden</title>'
                    b''
                    b''
                    b'
403 Forbidden
'
                    b''
                b''
                )

class InternalError(WebException):

    def default_response(self):
        return (b'500 Internal Server Error',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>500 Internal Server Error</title>'
                    b''
                    b''
                    b'
500 Internal Server Error
'
                    b''
                b''
                )

class NotFound(WebException):

    def default_response(self):
        return (b'404 Not Found',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>404 Page Not Found</title>'
                    b''
                    b''
                    b'
404 Page Not Found
'
                    b''
                b''
                )

class MethodNotAllowed(WebException):

    def default_response(self):
        return (b'405 Method Not Allowed',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>405 Method Not Allowed</title>'
                    b''
                    b''
                    b'
405 Method Not Allowed
'
                    b''
                b''
                )

class Redirect(WebException):

    def __init__(self, location, status_code, status_text):
        self.statusline = '{:d} {}'.format(status_code, status_text)
        self.location = location

    def location_header(self):
        return b'Location\0' + self.location.encode('ascii') + b'\0'

    def headers(self):
        return (b'Content-Type\0text/html\0'
                + self.location_header())

    def default_response(self):
        return (self.statusline, self.headers(),
                '<!DOCTYPE html>'
                ''
                    ''
                        '<title>{0.statusline}</title>'
                    ''
                    ''
                    '
{0.statusline}
'
                    'Follow'
                    ''
                ''.format(self)
                )

class CompletionRedirect(Redirect):
    """Temporary redirect which sends code 303

    With :param:`cookie` set it is often used for login forms. Without
    parameter set it is used to provide "success" page for various web forms
    and other non-idempotent actions
    """

    def __init__(self, location, cookie=None, *,
        status_code=303, status_text="See Other"):
        super().__init__(location,
            status_code=status_code, status_text=status_text)
        self.cookie = cookie

    def headers(self):
        sup = super().headers()
        if self.cookie is not None:
            sup += ('Set-Cookie\0'
                    + self.cookie.output(header='').encode('ascii') + '\0')
        return sup

class ChildNotFound(Exception):
    """Raised by resolve_local to notify that there is not such child"""

class BaseResolver(metaclass=abc.ABCMeta):

    def __init__(self, request):
        self.request = request

    @abc.abstractmethod
    def __next__(self):
        pass

    def __iter__(self):
        return self

    def child_fallback(self):
        raise NotFound()

    def resolve(self, root):
        self.resource = node = root
        for name in self:
            try:
                node = node.resolve_local(name)
            except ChildNotFound:
                node = self.child_fallback()
            kind = getattr(node, '_zweb', None)
            if kind is _PAGE_METHOD:
                return _dispatch_page(node, node.__self__, self)
            elif kind is _RESOURCE_METHOD:
                node, self.args = _dispatch_resource(node, node.__self__, self)
                self.resource = node
                if not isinstance(self, node.resolver_class):
                    newres =  node.resolver_class(self.request)
                    return newres.resolve(node)
            elif kind is _RESOURCE:
                pass
            else:
                raise NotFound()  # probably impossible but ...
        if(hasattr(node, 'index')
            and getattr(node.index, '_zweb', None) is _PAGE_METHOD):
            return _dispatch_page(node.index, node, self)
        raise NotFound()

class PathResolver(BaseResolver):

    def __init__(self, request):
        super().__init__(request)
        path = request.parsed_uri.path.strip('/')
        if path:
            self.args = path.split('/')
        else:
            self.args = ()

    def __next__(self):
        if not self.args:
            raise StopIteration()
        name, *args = self.args
        self.args = args
        return name

class MethodResolver(BaseResolver):

    def __init__(self, request):
        super().__init__(request)
        self.args = ()

    def __next__(self):
        return self.request.method.decode('ascii').upper()

    def child_fallback(self):
        raise MethodNotAllowed()

class InternalRedirect(Exception, metaclass=abc.ABCMeta):

    @abc.abstractmethod
    def update_request(self, request):
        pass

class PathRewrite(InternalRedirect):

    def __init__(self, new_path):
        self.new_path = new_path.encode('ascii')

    def update_request(self, request):
        request.uri = self.new_path
        del request.parsed_uri

class Resource(object):
    resolver_class = PathResolver
    _zweb = _RESOURCE

    def resolve_local(self, name):
        if not name.isidentifier() or name.startswith('_'):
            raise ChildNotFound()
        target = getattr(self, name, None)
        if target is None:
            raise ChildNotFound()
        kind = getattr(target, '_zweb', None)
        if kind is not None:
            return target
        raise ChildNotFound()

class Site(object):

    def __init__(self, *, request_class, resources=()):
        self.request_class = request_class
        self.resources = resources

    def _resolve(self, request):
        for i in self.resources:
            res = i.resolver_class(request)
            try:
                return res.resolve(i)
            except NotFound as e:
                continue
        else:
            raise NotFound()

    def _safe_dispatch(self, request):
        while True:
            try:
                result = self._resolve(request)
            except InternalRedirect as e:
                e.update_request(request)
                continue
            except Exception as e:
                if not isinstance(e, WebException):
                    log.exception("Can't process request %r", request)
                    e = InternalError(e)
                try:
                    return self.error_page(e)
                except Exception:
                    log.exception("Can't make error page for %r", e)
                    return e.default_response()
            else:
                return result

    def error_page(self, e):
        return e.default_response()

    def __call__(self, *args):
        request = self.request_class(*args)
        result = self._safe_dispatch(request)
        if hasattr(result, 'zerogw_response'):
            return result.zerogw_response()
        else:
            return result

def _dispatch_resource(fun, self, resolver):
    deco = getattr(fun, '_zweb_deco', None)
    if deco is not None:
        return deco(self, resolver,
            partial(fun._zweb_deco_callee, self, resolver),
            *resolver.args, **resolver.request.form_arguments)
    else:
        try:
            args, tail, kw = fun._zweb_sig(resolver,
                *resolver.args, **resolver.request.form_arguments)
        except (TypeError, ValueError) as e:
            log.debug("Signature mismatch %r %r",
                resolver.args, resolver.request.form_arguments,
                exc_info=e)  # debug
            raise NotFound()
        else:
            return fun(*args, **kw), tail

class ReprHack(str):
    __slots__ = ()
    def __repr__(self):
        return str(self)

def _compile_signature(fun, partial):
    sig = inspect.signature(fun)
    fun_params = [
        inspect.Parameter('resolver',
            kind=inspect.Parameter.POSITIONAL_OR_KEYWORD)]
    args = []
    kwargs = []
    vars = {
        '__empty__': object(),
        }
    lines = []
    annotations = {}
    self = True
    varkw = False
    varpos = False

    for name, param in sig.parameters.items():
        ann = param.annotation
        if param.default is not inspect.Parameter.empty:
            vars[name + '_def'] = param.default
            defname = ReprHack('__empty__')
        else:
            defname = inspect.Parameter.empty
        if ann is not inspect.Parameter.empty:
            if isinstance(ann, type) and issubclass(ann, Sticker):
                lines.append('  {0} = {0}_create(resolver)'.format(name))
                vars[name + '_create'] = ann.create
            else:
                lines.append('  if {0} is __empty__:'.format(name))
                lines.append('    {0} = {0}_def'.format(name))
                lines.append('  else:')
                if isinstance(ann, type) and ann.__module__ == 'builtins':
                    lines.append('    {0} = {1}({0})'.format(
                        name, ann.__name__))
                    fun_params.append(param.replace(
                        default=defname))
                else:
                    lines.append('    {0} = {0}_type({0})'.format(name))
                    vars[name + '_type'] = ann
                    fun_params.append(param.replace(
                        annotation=ReprHack(name + '_type'),
                        default=defname))
        elif not self:
            fun_params.append(param.replace(default=defname))

        if param.kind == inspect.Parameter.VAR_KEYWORD:
            varkw = True
        elif param.kind == inspect.Parameter.VAR_POSITIONAL:
            varpos = name
        if param.kind == inspect.Parameter.KEYWORD_ONLY:
            kwargs.append('{0!r}: {0}'.format(name))
        elif not self:
            args.append(name)
        if self:
            self = False
    if not varpos and partial:
        fun_params.append(inspect.Parameter('__tail__',
            kind=inspect.Parameter.VAR_POSITIONAL))
    if not varkw:
        fun_params.append(inspect.Parameter('__kw__',
            kind=inspect.Parameter.VAR_KEYWORD))
    funsig = inspect.Signature(fun_params)
    lines.insert(0, 'def __sig__{}:'.format(funsig))
    if len(args) == 1:
        args = args[0] + ','
    else:
        args = ', '.join(args)
    if varpos:
        lines.append('  return ({}) + {}, (), {{{}}}'.format(
            args, varpos,  ', '.join(kwargs)))
    elif partial:
        lines.append('  return ({}), __tail__, {{{}}}'.format(
            args, ', '.join(kwargs)))
    else:
        lines.append('  return ({}), (), {{{}}}'.format(
            args, ', '.join(kwargs)))
    code = compile('\n'.join(lines), '__sig__', 'exec')
    exec(code, vars)
    return vars['__sig__']

def resource(fun):
    """Decorator to denote a method which returns resource to be traversed"""
    fun._zweb = _RESOURCE_METHOD
    fun._zweb_sig = _compile_signature(fun, partial=True)
    return fun

def _dispatch_page(fun, self, resolver):
    deco = getattr(fun, '_zweb_deco', None)
    if deco is not None:
        result = deco(self, resolver,
            partial(fun._zweb_deco_callee, self, resolver),
            *resolver.args, **resolver.request.form_arguments)
    else:
        try:
            args, tail, kw = fun._zweb_sig(resolver,
                *resolver.args, **resolver.request.form_arguments)
        except (TypeError, ValueError) as e:
            log.debug("Signature mismatch %r %r",
                resolver.args, resolver.request.form_arguments,
                exc_info=e)  # debug
            raise NotFound()
        else:
            result = fun(*args, **kw)
    for proc in fun._zweb_post:
        result = proc(self, resolver, result)
    return result

def page(fun):
    """Decorator to denote a method which returns some result to the user"""
    if not hasattr(fun, '_zweb_post'):
        fun._zweb_post = []
    fun._zweb = _PAGE_METHOD
    fun._zweb_sig = _compile_signature(fun, partial=False)
    return fun

def postprocessor(fun):
    if not hasattr(fun, '_zweb_post'):
        fun._zweb_post = []
    def wrapper(proc):
        fun._zweb_post.append(proc)
        return fun
    return wrapper

def decorator(fun):
    def wrapper(parser):
        olddec = getattr(fun, '_zweb_deco', None)
        oldcallee = getattr(fun, '_zweb_deco_callee', None)
        if olddec is None:
            def callee(self, resolver, *args, **kw):
                try:
                    args, tail, kw = fun._zweb_sig(resolver, *args, **kw)
                except (TypeError, ValueError) as e:
                    log.debug("Signature mismatch %r %r", args, kw,
                        exc_info=e)  # debug
                    raise NotFound()
                else:
                    return fun(self, *args, **kw)
        else:
            def callee(self, resolver, *args, **kw):
                return olddec(self, resolver,
                    partial(oldcallee, self, resolver),
                    *args, **kw)

        fun._zweb_deco = parser
        fun._zweb_deco_callee = callee
        return fun
    return wrapper

class Sticker(metaclass=abc.ABCMeta):
    """
    An object which is automatically put into arguments in the view if
    specified in annotation
    """
    __superseded = {}

    @classmethod
    @abc.abstractmethod
    def create(cls, resolver):
        """Creates an object of this class based on resolver"""

    @classmethod
    @abc.abstractmethod
    def create(cls, resolver):
        """Creates an object of this class based on resolver"""

    @classmethod
    def supersede(cls, sub):
        oldsup = cls.__superseeded.get(cls, None)
        if oldsup is not None:
            if issubclass(sub, oldsup):
                pass  # just supersede it again
            elif issubclass(oldsup, sub):
                return  # already superseeded by more specific subclass
            else:
                raise RuntimeError("{!r} is already superseeded by {!r}"
                    .format(cls, oldsup))

        super().register(sub)
        cls.__superseded[cls] = sub

<commit_msg>Fix cookie in CompletionRedirect<commit_after>import abc
import logging
import inspect
from urllib.parse import urlparse, parse_qsl
from itertools import zip_longest
from collections import OrderedDict
from functools import partial

from .util import cached_property

log = logging.getLogger(__name__)
sentinel = object()

_PAGE_METHOD = object()
_RESOURCE_METHOD = object()
_RESOURCE = object()
_FORM_CTYPE = b'application/x-www-form-urlencoded'

class LegacyMultiDict(object):
    """Utilitary class which wrap dict to make it suitable for old utilities
    like wtforms"""

    def __init__(self, dic):
        self._dic = dic

    def getlist(self, k):
        return [self._dic[k]]

    def __contains__(self, k):
        return k in self._dic

    def __iter__(self):
        for k in self._dic:
            yield k

class Request(object):
    __slots__ = ('__dict__',
        # the excpected property names and names of respective zerogw items
        'uri',           # !Uri
        'content_type',  # !Header Content-Type
        'cookie',        # !Header Cookie
        'body',          # !PostBody
        )

    @cached_property
    def parsed_uri(self):
        return urlparse(self.uri.decode('ascii'))

    @cached_property
    def form_arguments(self):
        arguments = {}
        if hasattr(self, 'uri'):
            arguments.update(parse_qsl(self.parsed_uri.query))
        body = getattr(self, 'body', None)
        if body and getattr(self, 'content_type', None) == _FORM_CTYPE:
            arguments.update(parse_qsl(self.body.decode('ascii')))
        return arguments

    @cached_property
    def legacy_arguments(self):
        return LegacyMultiDict(self.form_arguments)

    @cached_property
    def cookies(self):
        cobj = SimpleCookie(self.cookie.decode('ascii', 'ignore'))
        return dict((k, cobj[k].value) for k in cobj)

class WebException(Exception):
    """Base for all exceptions which render error code (and page) to client"""

    @abc.abstractmethod
    def default_response(self):
        pass

class Forbidden(WebException):

    def default_response(self):
        return (b'403 Forbidden',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>403 Forbidden</title>'
                    b''
                    b''
                    b'
403 Forbidden
'
                    b''
                b''
                )

class InternalError(WebException):

    def default_response(self):
        return (b'500 Internal Server Error',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>500 Internal Server Error</title>'
                    b''
                    b''
                    b'
500 Internal Server Error
'
                    b''
                b''
                )

class NotFound(WebException):

    def default_response(self):
        return (b'404 Not Found',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>404 Page Not Found</title>'
                    b''
                    b''
                    b'
404 Page Not Found
'
                    b''
                b''
                )

class MethodNotAllowed(WebException):

    def default_response(self):
        return (b'405 Method Not Allowed',
                b'Content-Type\0text/html\0',
                b'<!DOCTYPE html>'
                b''
                    b''
                        b'<title>405 Method Not Allowed</title>'
                    b''
                    b''
                    b'
405 Method Not Allowed
'
                    b''
                b''
                )

class Redirect(WebException):

    def __init__(self, location, status_code, status_text):
        self.statusline = '{:d} {}'.format(status_code, status_text)
        self.location = location

    def location_header(self):
        return b'Location\0' + self.location.encode('ascii') + b'\0'

    def headers(self):
        return (b'Content-Type\0text/html\0'
                + self.location_header())

    def default_response(self):
        return (self.statusline, self.headers(),
                '<!DOCTYPE html>'
                ''
                    ''
                        '<title>{0.statusline}</title>'
                    ''
                    ''
                    '
{0.statusline}
'
                    'Follow'
                    ''
                ''.format(self)
                )

class CompletionRedirect(Redirect):
    """Temporary redirect which sends code 303

    With :param:`cookie` set it is often used for login forms. Without
    parameter set it is used to provide "success" page for various web forms
    and other non-idempotent actions
    """

    def __init__(self, location, cookie=None, *,
        status_code=303, status_text="See Other"):
        super().__init__(location,
            status_code=status_code, status_text=status_text)
        self.cookie = cookie

    def headers(self):
        sup = super().headers()
        if self.cookie is not None:
            sup += (b'Set-Cookie\0'
                    + self.cookie.output(header='').encode('ascii') + b'\0')
        return sup

class ChildNotFound(Exception):
    """Raised by resolve_local to notify that there is not such child"""

class BaseResolver(metaclass=abc.ABCMeta):

    def __init__(self, request):
        self.request = request

    @abc.abstractmethod
    def __next__(self):
        pass

    def __iter__(self):
        return self

    def child_fallback(self):
        raise NotFound()

    def resolve(self, root):
        self.resource = node = root
        for name in self:
            try:
                node = node.resolve_local(name)
            except ChildNotFound:
                node = self.child_fallback()
            kind = getattr(node, '_zweb', None)
            if kind is _PAGE_METHOD:
                return _dispatch_page(node, node.__self__, self)
            elif kind is _RESOURCE_METHOD:
                node, self.args = _dispatch_resource(node, node.__self__, self)
                self.resource = node
                if not isinstance(self, node.resolver_class):
                    newres =  node.resolver_class(self.request)
                    return newres.resolve(node)
            elif kind is _RESOURCE:
                pass
            else:
                raise NotFound()  # probably impossible but ...
        if(hasattr(node, 'index')
            and getattr(node.index, '_zweb', None) is _PAGE_METHOD):
            return _dispatch_page(node.index, node, self)
        raise NotFound()

class PathResolver(BaseResolver):

    def __init__(self, request):
        super().__init__(request)
        path = request.parsed_uri.path.strip('/')
        if path:
            self.args = path.split('/')
        else:
            self.args = ()

    def __next__(self):
        if not self.args:
            raise StopIteration()
        name, *args = self.args
        self.args = args
        return name

class MethodResolver(BaseResolver):

    def __init__(self, request):
        super().__init__(request)
        self.args = ()

    def __next__(self):
        return self.request.method.decode('ascii').upper()

    def child_fallback(self):
        raise MethodNotAllowed()

class InternalRedirect(Exception, metaclass=abc.ABCMeta):

    @abc.abstractmethod
    def update_request(self, request):
        pass

class PathRewrite(InternalRedirect):

    def __init__(self, new_path):
        self.new_path = new_path.encode('ascii')

    def update_request(self, request):
        request.uri = self.new_path
        del request.parsed_uri

class Resource(object):
    resolver_class = PathResolver
    _zweb = _RESOURCE

    def resolve_local(self, name):
        if not name.isidentifier() or name.startswith('_'):
            raise ChildNotFound()
        target = getattr(self, name, None)
        if target is None:
            raise ChildNotFound()
        kind = getattr(target, '_zweb', None)
        if kind is not None:
            return target
        raise ChildNotFound()

class Site(object):

    def __init__(self, *, request_class, resources=()):
        self.request_class = request_class
        self.resources = resources

    def _resolve(self, request):
        for i in self.resources:
            res = i.resolver_class(request)
            try:
                return res.resolve(i)
            except NotFound as e:
                continue
        else:
            raise NotFound()

    def _safe_dispatch(self, request):
        while True:
            try:
                result = self._resolve(request)
            except InternalRedirect as e:
                e.update_request(request)
                continue
            except Exception as e:
                if not isinstance(e, WebException):
                    log.exception("Can't process request %r", request)
                    e = InternalError(e)
                try:
                    return self.error_page(e)
                except Exception:
                    log.exception("Can't make error page for %r", e)
                    return e.default_response()
            else:
                return result

    def error_page(self, e):
        return e.default_response()

    def __call__(self, *args):
        request = self.request_class(*args)
        result = self._safe_dispatch(request)
        if hasattr(result, 'zerogw_response'):
            return result.zerogw_response()
        else:
            return result

def _dispatch_resource(fun, self, resolver):
    deco = getattr(fun, '_zweb_deco', None)
    if deco is not None:
        return deco(self, resolver,
            partial(fun._zweb_deco_callee, self, resolver),
            *resolver.args, **resolver.request.form_arguments)
    else:
        try:
            args, tail, kw = fun._zweb_sig(resolver,
                *resolver.args, **resolver.request.form_arguments)
        except (TypeError, ValueError) as e:
            log.debug("Signature mismatch %r %r",
                resolver.args, resolver.request.form_arguments,
                exc_info=e)  # debug
            raise NotFound()
        else:
            return fun(*args, **kw), tail

class ReprHack(str):
    __slots__ = ()
    def __repr__(self):
        return str(self)

def _compile_signature(fun, partial):
    sig = inspect.signature(fun)
    fun_params = [
        inspect.Parameter('resolver',
            kind=inspect.Parameter.POSITIONAL_OR_KEYWORD)]
    args = []
    kwargs = []
    vars = {
        '__empty__': object(),
        }
    lines = []
    annotations = {}
    self = True
    varkw = False
    varpos = False

    for name, param in sig.parameters.items():
        ann = param.annotation
        if param.default is not inspect.Parameter.empty:
            vars[name + '_def'] = param.default
            defname = ReprHack('__empty__')
        else:
            defname = inspect.Parameter.empty
        if ann is not inspect.Parameter.empty:
            if isinstance(ann, type) and issubclass(ann, Sticker):
                lines.append('  {0} = {0}_create(resolver)'.format(name))
                vars[name + '_create'] = ann.create
            else:
                lines.append('  if {0} is __empty__:'.format(name))
                lines.append('    {0} = {0}_def'.format(name))
                lines.append('  else:')
                if isinstance(ann, type) and ann.__module__ == 'builtins':
                    lines.append('    {0} = {1}({0})'.format(
                        name, ann.__name__))
                    fun_params.append(param.replace(
                        default=defname))
                else:
                    lines.append('    {0} = {0}_type({0})'.format(name))
                    vars[name + '_type'] = ann
                    fun_params.append(param.replace(
                        annotation=ReprHack(name + '_type'),
                        default=defname))
        elif not self:
            fun_params.append(param.replace(default=defname))

        if param.kind == inspect.Parameter.VAR_KEYWORD:
            varkw = True
        elif param.kind == inspect.Parameter.VAR_POSITIONAL:
            varpos = name
        if param.kind == inspect.Parameter.KEYWORD_ONLY:
            kwargs.append('{0!r}: {0}'.format(name))
        elif not self:
            args.append(name)
        if self:
            self = False
    if not varpos and partial:
        fun_params.append(inspect.Parameter('__tail__',
            kind=inspect.Parameter.VAR_POSITIONAL))
    if not varkw:
        fun_params.append(inspect.Parameter('__kw__',
            kind=inspect.Parameter.VAR_KEYWORD))
    funsig = inspect.Signature(fun_params)
    lines.insert(0, 'def __sig__{}:'.format(funsig))
    if len(args) == 1:
        args = args[0] + ','
    else:
        args = ', '.join(args)
    if varpos:
        lines.append('  return ({}) + {}, (), {{{}}}'.format(
            args, varpos,  ', '.join(kwargs)))
    elif partial:
        lines.append('  return ({}), __tail__, {{{}}}'.format(
            args, ', '.join(kwargs)))
    else:
        lines.append('  return ({}), (), {{{}}}'.format(
            args, ', '.join(kwargs)))
    code = compile('\n'.join(lines), '__sig__', 'exec')
    exec(code, vars)
    return vars['__sig__']

def resource(fun):
    """Decorator to denote a method which returns resource to be traversed"""
    fun._zweb = _RESOURCE_METHOD
    fun._zweb_sig = _compile_signature(fun, partial=True)
    return fun

def _dispatch_page(fun, self, resolver):
    deco = getattr(fun, '_zweb_deco', None)
    if deco is not None:
        result = deco(self, resolver,
            partial(fun._zweb_deco_callee, self, resolver),
            *resolver.args, **resolver.request.form_arguments)
    else:
        try:
            args, tail, kw = fun._zweb_sig(resolver,
                *resolver.args, **resolver.request.form_arguments)
        except (TypeError, ValueError) as e:
            log.debug("Signature mismatch %r %r",
                resolver.args, resolver.request.form_arguments,
                exc_info=e)  # debug
            raise NotFound()
        else:
            result = fun(*args, **kw)
    for proc in fun._zweb_post:
        result = proc(self, resolver, result)
    return result

def page(fun):
    """Decorator to denote a method which returns some result to the user"""
    if not hasattr(fun, '_zweb_post'):
        fun._zweb_post = []
    fun._zweb = _PAGE_METHOD
    fun._zweb_sig = _compile_signature(fun, partial=False)
    return fun

def postprocessor(fun):
    if not hasattr(fun, '_zweb_post'):
        fun._zweb_post = []
    def wrapper(proc):
        fun._zweb_post.append(proc)
        return fun
    return wrapper

def decorator(fun):
    def wrapper(parser):
        olddec = getattr(fun, '_zweb_deco', None)
        oldcallee = getattr(fun, '_zweb_deco_callee', None)
        if olddec is None:
            def callee(self, resolver, *args, **kw):
                try:
                    args, tail, kw = fun._zweb_sig(resolver, *args, **kw)
                except (TypeError, ValueError) as e:
                    log.debug("Signature mismatch %r %r", args, kw,
                        exc_info=e)  # debug
                    raise NotFound()
                else:
                    return fun(self, *args, **kw)
        else:
            def callee(self, resolver, *args, **kw):
                return olddec(self, resolver,
                    partial(oldcallee, self, resolver),
                    *args, **kw)

        fun._zweb_deco = parser
        fun._zweb_deco_callee = callee
        return fun
    return wrapper

class Sticker(metaclass=abc.ABCMeta):
    """
    An object which is automatically put into arguments in the view if
    specified in annotation
    """
    __superseded = {}

    @classmethod
    @abc.abstractmethod
    def create(cls, resolver):
        """Creates an object of this class based on resolver"""

    @classmethod
    @abc.abstractmethod
    def create(cls, resolver):
        """Creates an object of this class based on resolver"""

    @classmethod
    def supersede(cls, sub):
        oldsup = cls.__superseeded.get(cls, None)
        if oldsup is not None:
            if issubclass(sub, oldsup):
                pass  # just supersede it again
            elif issubclass(oldsup, sub):
                return  # already superseeded by more specific subclass
            else:
                raise RuntimeError("{!r} is already superseeded by {!r}"
                    .format(cls, oldsup))

        super().register(sub)
        cls.__superseded[cls] = sub

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import json
from django import forms
from django.template.loader import render_to_string
from django.utils import six
from django.utils.translation import ugettext_lazy as _
from .conf import settings

class GeopositionWidget(forms.MultiWidget):
    def __init__(self, attrs=None):
        widgets = (
            forms.TextInput(),
            forms.TextInput(),
        )
        super(GeopositionWidget, self).__init__(widgets, attrs)

    def decompress(self, value):
        if isinstance(value, six.text_type):
            return value.rsplit(',')
        if value:
            return [value.latitude, value.longitude]
        return [None,None]

    def format_output(self, rendered_widgets):
        return render_to_string('geoposition/widgets/geoposition.html', {
            'latitude': {
                'html': rendered_widgets[0],
                'label': _("latitude"),
            },
            'longitude': {
                'html': rendered_widgets[1],
                'label': _("longitude"),
            },
            'config': {
                'map_widget_height': settings.GEOPOSITION_MAP_WIDGET_HEIGHT,
                'map_options': json.dumps(settings.GEOPOSITION_MAP_OPTIONS),
                'marker_options': json.dumps(settings.GEOPOSITION_MARKER_OPTIONS),
            }
        })

    class Media:
        js = (
            '//maps.google.com/maps/api/js?sensor=false',
            'geoposition/geoposition.js',
        )
        css = {
            'all': ('geoposition/geoposition.css',)
        }
<commit_msg>Fix google maps<commit_after>from __future__ import unicode_literals

import json
from django import forms
from django.template.loader import render_to_string
from django.utils import six
from django.utils.translation import ugettext_lazy as _
from .conf import settings

class GeopositionWidget(forms.MultiWidget):
    def __init__(self, attrs=None):
        widgets = (
            forms.TextInput(),
            forms.TextInput(),
        )
        super(GeopositionWidget, self).__init__(widgets, attrs)

    def decompress(self, value):
        if isinstance(value, six.text_type):
            return value.rsplit(',')
        if value:
            return [value.latitude, value.longitude]
        return [None,None]

    def format_output(self, rendered_widgets):
        return render_to_string('geoposition/widgets/geoposition.html', {
            'latitude': {
                'html': rendered_widgets[0],
                'label': _("latitude"),
            },
            'longitude': {
                'html': rendered_widgets[1],
                'label': _("longitude"),
            },
            'config': {
                'map_widget_height': settings.GEOPOSITION_MAP_WIDGET_HEIGHT,
                'map_options': json.dumps(settings.GEOPOSITION_MAP_OPTIONS),
                'marker_options': json.dumps(settings.GEOPOSITION_MARKER_OPTIONS),
            }
        })

    class Media:
        js = (
            '//maps.google.com/maps/api/js?sensor=false&v=3',
            'geoposition/geoposition.js',
        )
        css = {
            'all': ('geoposition/geoposition.css',)
        }
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
Module program description
	1. This program does
	2. The user interface is ....

"""

PORT=8080	#: Sets the default port

#HOST='0.0.0.0'  #: Accept anyhost
HOST='127.0.0.1' # Local host only

#DEBUG= True     #: True - Debug ON
DEBUG= False

from bottle import static_file, abort, redirect
from bottle import route, run, debug
from bottle import template, request, response, post, get
from bottle import validate, static_file

from  zoterotool import *

open_database("zotero.sqlite");

def link_tpl(url,name,conf=0):
    """
    Create html link code to URL
    Returns the html code to the link

    conf=0 : Any link
    conf=1 : Internal file

    """

    if conf ==1:
         url="/files/" + url

    link = '' + name + ''
    return link

def get_item_link(itemid):
    """
    Returns a html link to the file, given the fileID

    """
    # Get item attachment path
    #path =  get_item_attachment(itemid)
    path =  get_attachment(itemid)

    if path is not None:
        #path = os.path.join("/files", path)
        #path = "/files" + path

        name = os.path.split(path)[1]
        link = link_tpl( path, name,1 )
        #print link

        #link = '' + "file" + ""

        return link

    return None

def print_all_links():
    items =  get_items()

    linklist = ""

    for item in items:
        itemId, itemName = item
        #print str(itemId)
        #print itemName

        path =  get_item_attachment(itemId)
        if path is not None and path != -1 :
            #print "path = " +  path
            link = link_tpl(path,itemName,1)

            linklist= linklist + link + "\n " + "
"
            #print link

    return linklist

def link_list_tpl ( url_list ):
    """
    Generates a html code of a  list of links
    given a list of URL

    url_list = [ [ "URL1", "Description1"] , [ "URL2", "description2" ] .... ]

    """

    html = ""

    for url_ in url_list:

        url, name = url_
        link = link_tpl(url, name)
        html = html + "*" + link  + "
\n"

    return html

@route('/index')
@route('/')
def index():

    link_list   =  link_list_tpl(\
            [\
            [ "/items", "Items"             ] ,\
            [ "/tags", "Tags"               ] ,\
            [ "/collections", "Collections" ] ,\
            [ "/help","Help"                ] ])

    button = '''

    <form action="/updatelib" method="post">
        <input value="Update Library" type="submit" />
    </form>
    '''

    content_ =  link_list + button
    return template("base.html", subtitle="Options:", content= content_ , backlink = "index" )

@post('/updatelib')
def updatelibrary():
    global conn

    os.system("./update-data.sh")
    close_database()
    open_database("zotero.sqlite");
    redirect("/index")

@route('/items')
def all_items():
    """
    In this page all Zotero collection
    items are printed with a download
    link.

    """

    links =  print_all_links()
    return template("base.html", subtitle="Items", content= links, backlink="index" )
#    return links

@route('/collections')
def all_collections():
    """
    Show the user all Zotero collections
    with a link to the items in the collections

    Page code
    ---------
    link-to-collection-id1    http://site-host/collectionid/id1
    link-to-collection-id2 :  http://site-host/collectionid/id2
    ...
    ...

    """

    collections = get_collections()

    html = ""

    for coll in collections:
        collid, collname = coll

        #print "collid " + str(collid)

        url   = "/collectionid/" + str(collid)
        link  = link_tpl(url,collname)

        html = html + link + "
\n"

#   links =  print_all_links()
    return template("base.html", subtitle="Collections", content= html, backlink="index" )
#    return html

@route('/collectionid/<collid:int>')
def show_collection(collid):

    collname = get_collection_name(collid)

    itemIDs= get_item_from_collections(collid)

    #print itemIDs

    html = ""

    #print "show_collection Debug Trace ----------__"

    for itemid in itemIDs:

        link = get_item_link(itemid)

        if link is not None:
            html = html + link + "
\n"

            #print "link " + link
            #print "itemid " + str(itemid)

    subtilte_ = "Collection: " + collname
    return template("base.html", subtitle= subtilte_ , content= html, backlink="collections" )

@route('/fileid/<itemid:int>')
def retrive_file(itemid):
    """
    Retrives file that matches itemid

    """

    path = get_attachment(itemid)
    if path is not None:
        path_, file_ = os.path.split(path)

        #print path_
        #print file_

        return static_file(file_, path_, download = file_ )
#       return "File was  " + str(itemid) + " " + path
    else:
        return "Error: File not found"

@route('/files/<path:path>')
def callback(path):

    #path=os.path.join("/",path)
    print path

    #print "path =" + path
    if os.path.isfile( path ):
        path_, file_ = os.path.split(path)

        print "retriving"
        print "path " + path_
        print "filename =" + file_

        return static_file(file_, path_ )
    else:
        return "Error: File don't exist on server."

@route('/tags')
def show_tags():
    """
    In this page all tags are showed
    when some tag is clicked only the files
    related to this tag will be showed to the user

    """

    html = ""

    tags = get_tags()
    for tag in tags:

        tagid, tagname = tag

        url =   "/tagid/" + str(tagid)
        #print url
        link = link_tpl(url,tagname)

        html = html + link + "
\n"

    return template("base.html", subtitle="Tags", content= html, backlink="index" )
#    return html

@route('/tagid/<tagid:int>')
def show_tagid(tagid):

    tagname = get_tagname(tagid)

    itemIDs = filter_tag(tagid)

    html = ""
    for itemid in itemIDs:

        link = get_item_link(itemid)

        if link is not None:
            html = html + link + "
\n"

    subtilte_ = "Tag: " + tagname
    return template("base.html", subtitle= subtilte_ , content= html, backlink="tags" )

@route("/top")
def top():
    import subprocess

    response.content_type = "text/plain"
    return subprocess.check_output(["top", "-bn", "1"])

@route("/help")
def help():

    html = """
    The ZOTERO SERVER - Is a http web server that uses bottle framework.

    This simple and lightweight web server allows to access the Zotero

    data from anywhere, any device, tablet, smartphone, PC ...

    """

    return template("base.html", subtitle= "HELP" , content= html , backlink="index" )

@get('/favicon.ico')
def get_favicon():
#    return server_static('favicon.ico')
    return static_file('favicon.ico', "." )

# Run the server
if __name__=="__main__":
    run(host=HOST, port=PORT, debug=DEBUG, reloader=True)

<commit_msg>Added status interface<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
Module program description
	1. This program does
	2. The user interface is ....

"""

PORT=8080	#: Sets the default port

HOST='0.0.0.0'  #: Accept anyhost
#HOST='127.0.0.1' # Local host only

#DEBUG= True     #: True - Debug ON
DEBUG= False

from bottle import static_file, abort, redirect
from bottle import route, run, debug
from bottle import template, request, response, post, get
from bottle import validate, static_file

from  zoterotool import *

open_database("zotero.sqlite");

def link_tpl(url,name,conf=0):
    """
    Create html link code to URL
    Returns the html code to the link

    conf=0 : Any link
    conf=1 : Internal file

    """

    if conf ==1:
         url="/files/" + url

    link = '' + name + ''
    return link

def get_item_link(itemid):
    """
    Returns a html link to the file, given the fileID

    """
    # Get item attachment path
    #path =  get_item_attachment(itemid)
    path =  get_attachment(itemid)

    if path is not None:
        #path = os.path.join("/files", path)
        #path = "/files" + path

        name = os.path.split(path)[1]
        link = link_tpl( path, name,1 )
        #print link

        #link = '' + "file" + ""

        return link

    return None

def print_all_links():
    items =  get_items()

    linklist = ""

    for item in items:
        itemId, itemName = item
        #print str(itemId)
        #print itemName

        path =  get_item_attachment(itemId)
        if path is not None and path != -1 :
            #print "path = " +  path
            link = link_tpl(path,itemName,1)

            linklist= linklist + link + "\n " + "
"
            #print link

    return linklist

def link_list_tpl ( url_list ):
    """
    Generates a html code of a  list of links
    given a list of URL

    url_list = [ [ "URL1", "Description1"] , [ "URL2", "description2" ] .... ]

    """

    html = ""

    for url_ in url_list:

        url, name = url_
        link = link_tpl(url, name)
        html = html + "*" + link  + "
\n"

    return html

@route('/index')
@route('/')
def index():

    link_list   =  link_list_tpl(\
            [\
            [ "/items", "Items"             ] ,\
            [ "/tags", "Tags"               ] ,\
            [ "/collections", "Collections" ] ,\
            [ "/status","Server Status"     ] ,\
            [ "/help","Help"                ] ])

    button = '''

    <form action="/updatelib" method="post">
        <input value="Update Library" type="submit" />
    </form>
    '''

    content_ =  link_list + button
    return template("base.html", subtitle="Options:", content= content_ , backlink = "index" )

@post('/updatelib')
def updatelibrary():
    global conn

    os.system("./update-data.sh")
    close_database()
    open_database("zotero.sqlite");
    redirect("/index")

@route('/items')
def all_items():
    """
    In this page all Zotero collection
    items are printed with a download
    link.

    """

    links =  print_all_links()
    return template("base.html", subtitle="Items", content= links, backlink="index" )
#    return links

@route('/collections')
def all_collections():
    """
    Show the user all Zotero collections
    with a link to the items in the collections

    Page code
    ---------
    link-to-collection-id1    http://site-host/collectionid/id1
    link-to-collection-id2 :  http://site-host/collectionid/id2
    ...
    ...

    """

    collections = get_collections()

    html = ""

    for coll in collections:
        collid, collname = coll

        #print "collid " + str(collid)

        url   = "/collectionid/" + str(collid)
        link  = link_tpl(url,collname)

        html = html + link + "
\n"

#   links =  print_all_links()
    return template("base.html", subtitle="Collections", content= html, backlink="index" )
#    return html

@route('/collectionid/<collid:int>')
def show_collection(collid):

    collname = get_collection_name(collid)

    itemIDs= get_item_from_collections(collid)

    #print itemIDs

    html = ""

    #print "show_collection Debug Trace ----------__"

    for itemid in itemIDs:

        link = get_item_link(itemid)

        if link is not None:
            html = html + link + "
\n"

            #print "link " + link
            #print "itemid " + str(itemid)

    subtilte_ = "Collection: " + collname
    return template("base.html", subtitle= subtilte_ , content= html, backlink="collections" )

@route('/fileid/<itemid:int>')
def retrive_file(itemid):
    """
    Retrives file that matches itemid

    """

    path = get_attachment(itemid)
    if path is not None:
        path_, file_ = os.path.split(path)

        #print path_
        #print file_

        return static_file(file_, path_, download = file_ )
#       return "File was  " + str(itemid) + " " + path
    else:
        return "Error: File not found"

@route('/files/<path:path>')
def callback(path):

    #path=os.path.join("/",path)
    #print path

    #print "path =" + path
    if os.path.isfile( path ):
        path_, file_ = os.path.split(path)

        #print "retriving"
        #print "path " + path_
        #print "filename =" + file_

        return static_file(file_, path_ )
    else:
        return "Error: File don't exist on server."

@route('/tags')
def show_tags():
    """
    In this page all tags are showed
    when some tag is clicked only the files
    related to this tag will be showed to the user

    """

    html = ""

    tags = get_tags()
    for tag in tags:

        tagid, tagname = tag

        url =   "/tagid/" + str(tagid)
        #print url
        link = link_tpl(url,tagname)

        html = html + link + "
\n"

    return template("base.html", subtitle="Tags", content= html, backlink="index" )
#    return html

@route('/tagid/<tagid:int>')
def show_tagid(tagid):

    tagname = get_tagname(tagid)

    itemIDs = filter_tag(tagid)

    html = ""
    for itemid in itemIDs:

        link = get_item_link(itemid)

        if link is not None:
            html = html + link + "
\n"

    subtilte_ = "Tag: " + tagname
    return template("base.html", subtitle= subtilte_ , content= html, backlink="tags" )

@route("/status")
def status():
    """
    Shows the status of the server
    """
    import subprocess

    response.content_type = "text/plain"
    return subprocess.check_output(["cat", "/tmp/zotserver.log"])

@route("/help")
def help():

    html = """
    The ZOTERO SERVER - Is a http web server that uses bottle framework.

    This simple and lightweight web server allows to access the Zotero

    data from anywhere, any device, tablet, smartphone, PC ...

    """

    return template("base.html", subtitle= "HELP" , content= html , backlink="index" )

@get('/favicon.ico')
def get_favicon():
#    return server_static('favicon.ico')
    return static_file('favicon.ico', "." )

# Run the server
if __name__=="__main__":
    run(host=HOST, port=PORT, debug=DEBUG, reloader=True)

<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Script to parse StorCLI's JSON output and expose
MegaRAID health as Prometheus metrics.

Tested against StorCLI 'Ver 1.14.12 Nov 25, 2014'.

StorCLI reference manual:
http://docs.avagotech.com/docs/12352476

Advanced Software Options (ASO) not exposed as metrics currently.

JSON key abbreviations used by StorCLI are documented in the standard command
output, i.e.  when you omit the trailing 'J' from the command.

Formatting done with YAPF:
$ yapf -i --style '{COLUMN_LIMIT: 99}' storcli.py
"""

from __future__ import print_function
from datetime import datetime
import argparse
import collections
import json
import os
import shlex
import subprocess

DESCRIPTION = """Parses StorCLI's JSON output and exposes MegaRAID health as
    Prometheus metrics."""
VERSION = '0.0.3'

storcli_path = ''
metric_prefix = 'megaraid_'
metric_list = {}
metric_list = collections.defaultdict(list)

def main(args):
    """ main """
    global storcli_path
    storcli_path = args.storcli_path
    data = get_storcli_json('/cALL show all J')

    try:
        # All the information is collected underneath the Controllers key
        data = data['Controllers']

        for controller in data:
            response = controller['Response Data']
            if response['Version']['Driver Name'] == 'megaraid_sas':
                handle_megaraid_controller(response)
            elif response['Version']['Driver Name'] == 'mpt3sas':
                handle_sas_controller(response)
    except KeyError:
        pass

    print_all_metrics(metric_list)

def handle_sas_controller(response):
    (controller_index, baselabel) = get_basic_controller_info(response)
    add_metric('healthy', baselabel, int(response['Status']['Controller Status'] == 'OK'))
    add_metric('ports', baselabel, response['HwCfg']['Backend Port Count'])
    try:
        # The number of physical disks is half of the number of items in this dict
        # Every disk is listed twice - once for basic info, again for detailed info
        add_metric('physical_drives', baselabel,
                   len(response['Physical Device Information'].keys()) / 2)
    except AttributeError:
        pass
    # Split up string to not trigger CodeSpell issues
    add_metric('temperature', baselabel,
               int(response['HwCfg']['ROC temperature(Degree Celc' + 'ius)']))
    for key, basic_disk_info in response['Physical Device Information'].items():
        if 'Detailed Information' in key:
            continue
        create_metrcis_of_physical_drive(basic_disk_info[0],
                                         response['Physical Device Information'], controller_index)

def handle_megaraid_controller(response):
    (controller_index, baselabel) = get_basic_controller_info(response)

    # BBU Status Optimal value is 0 for cachevault and 32 for BBU
    add_metric('battery_backup_healthy', baselabel,
               int(response['Status']['BBU Status'] in [0, 32]))
    add_metric('degraded', baselabel, int(response['Status']['Controller Status'] == 'Degraded'))
    add_metric('failed', baselabel, int(response['Status']['Controller Status'] == 'Failed'))
    add_metric('healthy', baselabel, int(response['Status']['Controller Status'] == 'Optimal'))
    add_metric('drive_groups', baselabel, response['Drive Groups'])
    add_metric('virtual_drives', baselabel, response['Virtual Drives'])
    add_metric('physical_drives', baselabel, response['Physical Drives'])
    add_metric('ports', baselabel, response['HwCfg']['Backend Port Count'])
    add_metric('scheduled_patrol_read', baselabel,
               int('hrs' in response['Scheduled Tasks']['Patrol Read Reoccurrence']))
    add_metric('temperature', baselabel, int(response['HwCfg']['ROC temperature(Degree Celsius)']))
    for cvidx, cvinfo in enumerate(response['Cachevault_Info']):
        add_metric('cv_temperature', baselabel + ',cvidx="' + str(cvidx) + '"', int(cvinfo['Temp'].replace('C','')))

    time_difference_seconds = -1
    system_time = datetime.strptime(response['Basics'].get('Current System Date/time'),
                                    "%m/%d/%Y, %H:%M:%S")
    controller_time = datetime.strptime(response['Basics'].get('Current Controller Date/Time'),
                                        "%m/%d/%Y, %H:%M:%S")
    if system_time and controller_time:
        time_difference_seconds = abs(system_time - controller_time).seconds
        add_metric('time_difference', baselabel, time_difference_seconds)

    for virtual_drive in response['VD LIST']:
        vd_position = virtual_drive.get('DG/VD')
        drive_group, volume_group = -1, -1
        if vd_position:
            drive_group = vd_position.split('/')[0]
            volume_group = vd_position.split('/')[1]
        vd_baselabel = 'controller="{0}",DG="{1}",VG="{2}"'.format(controller_index, drive_group,
                                                                volume_group)
        vd_info_label = vd_baselabel + ',name="{0}",cache="{1}",type="{2}",state="{3}"'.format(
            str(virtual_drive.get('Name')).strip(),
            str(virtual_drive.get('Cache')).strip(),
            str(virtual_drive.get('TYPE')).strip(),
            str(virtual_drive.get('State')).strip())
        add_metric('vd_info', vd_info_label, 1)

    if response['Physical Drives'] > 0:
        data = get_storcli_json('/cALL/eALL/sALL show all J')
        drive_info = data['Controllers'][controller_index]['Response Data']
    for physical_drive in response['PD LIST']:
        create_metrcis_of_physical_drive(physical_drive, drive_info, controller_index)

def get_basic_controller_info(response):
    controller_index = response['Basics']['Controller']
    baselabel = 'controller="{0}"'.format(controller_index)

    controller_info_label = baselabel + ',model="{0}",serial="{1}",fwversion="{2}"'.format(
        str(response['Basics']['Model']).strip(),
        str(response['Basics']['Serial Number']).strip(),
        str(response['Version']['Firmware Version']).strip(),
    )
    add_metric('controller_info', controller_info_label, 1)

    return (controller_index, baselabel)

def create_metrcis_of_physical_drive(physical_drive, detailed_info_array, controller_index):
    enclosure = physical_drive.get('EID:Slt').split(':')[0]
    slot = physical_drive.get('EID:Slt').split(':')[1]

    pd_baselabel = 'controller="{0}",enclosure="{1}",slot="{2}"'.format(controller_index, enclosure,
                                                                     slot)
    pd_info_label = pd_baselabel + \
        ',disk_id="{0}",interface="{1}",media="{2}",model="{3}",DG="{4}",state="{5}"'.format(
            str(physical_drive.get('DID')).strip(),
            str(physical_drive.get('Intf')).strip(),
            str(physical_drive.get('Med')).strip(),
            str(physical_drive.get('Model')).strip(),
            str(physical_drive.get('DG')).strip(),
            str(physical_drive.get('State')).strip())

    drive_identifier = 'Drive /c' + str(controller_index) + '/e' + str(enclosure) + '/s' + str(
        slot)
    if enclosure == ' ':
        drive_identifier = 'Drive /c' + str(controller_index) + '/s' + str(slot)
    try:
        info = detailed_info_array[drive_identifier + ' - Detailed Information']
        state = info[drive_identifier + ' State']
        attributes = info[drive_identifier + ' Device attributes']
        settings = info[drive_identifier + ' Policies/Settings']

        add_metric('pd_shield_counter', pd_baselabel, state['Shield Counter'])
        add_metric('pd_media_errors', pd_baselabel, state['Media Error Count'])
        add_metric('pd_other_errors', pd_baselabel, state['Other Error Count'])
        add_metric('pd_predictive_errors', pd_baselabel, state['Predictive Failure Count'])
        add_metric('pd_smart_alerted', pd_baselabel,
                   int(state['S.M.A.R.T alert flagged by drive'] == 'Yes'))
        add_metric('pd_link_speed_gbps', pd_baselabel, attributes['Link Speed'].split('.')[0])
        add_metric('pd_device_speed_gbps', pd_baselabel, attributes['Device Speed'].split('.')[0])
        add_metric('pd_commissioned_spare', pd_baselabel,
                   int(settings['Commissioned Spare'] == 'Yes'))
        add_metric('pd_emergency_spare', pd_baselabel, int(settings['Emergency Spare'] == 'Yes'))
        pd_info_label += ',firmware="{0}"'.format(attributes['Firmware Revision'].strip())
    except KeyError:
        pass
    add_metric('pd_info', pd_info_label, 1)

def add_metric(name, labels, value):
    global metric_list
    try:
        metric_list[name].append({
            'labels': labels,
            'value': float(value),
        })
    except ValueError:
        pass

def print_all_metrics(metrics):
    for metric, measurements in metrics.items():
        print('# HELP {0}{1} MegaRAID {2}'.format(metric_prefix, metric, metric.replace('_', ' ')))
        print('# TYPE {0}{1} gauge'.format(metric_prefix, metric))
        for measurement in measurements:
            if measurement['value'] != 'Unknown':
                print('{0}{1}{2} {3}'.format(metric_prefix, metric, '{' + measurement['labels'] + '}',
                                         measurement['value']))

def get_storcli_json(storcli_args):
    """Get storcli output in JSON format."""
    # Check if storcli is installed and executable
    if not (os.path.isfile(storcli_path) and os.access(storcli_path, os.X_OK)):
        SystemExit(1)
    storcli_cmd = shlex.split(storcli_path + ' ' + storcli_args)
    proc = subprocess.Popen(
        storcli_cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output_json = proc.communicate()[0]
    data = json.loads(output_json.decode("utf-8"))

    if data["Controllers"][0]["Command Status"]["Status"] != "Success":
        SystemExit(1)
    return data

if __name__ == "__main__":
    PARSER = argparse.ArgumentParser(
        description=DESCRIPTION, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    PARSER.add_argument(
        '--storcli_path', default='/opt/MegaRAID/storcli/storcli64', help='path to StorCLi binary')
    PARSER.add_argument('--version', action='version', version='%(prog)s {0}'.format(VERSION))
    ARGS = PARSER.parse_args()

    main(ARGS)
<commit_msg>fix for 'Celsius' spelling problem in storcli.py (#1408)<commit_after>#!/usr/bin/env python3
"""
Script to parse StorCLI's JSON output and expose
MegaRAID health as Prometheus metrics.

Tested against StorCLI 'Ver 1.14.12 Nov 25, 2014'.

StorCLI reference manual:
http://docs.avagotech.com/docs/12352476

Advanced Software Options (ASO) not exposed as metrics currently.

JSON key abbreviations used by StorCLI are documented in the standard command
output, i.e.  when you omit the trailing 'J' from the command.

Formatting done with YAPF:
$ yapf -i --style '{COLUMN_LIMIT: 99}' storcli.py
"""

from __future__ import print_function
from datetime import datetime
import argparse
import collections
import json
import os
import shlex
import subprocess

DESCRIPTION = """Parses StorCLI's JSON output and exposes MegaRAID health as
    Prometheus metrics."""
VERSION = '0.0.3'

storcli_path = ''
metric_prefix = 'megaraid_'
metric_list = {}
metric_list = collections.defaultdict(list)

def main(args):
    """ main """
    global storcli_path
    storcli_path = args.storcli_path
    data = get_storcli_json('/cALL show all J')

    try:
        # All the information is collected underneath the Controllers key
        data = data['Controllers']

        for controller in data:
            response = controller['Response Data']

            handle_common_controller(response)
            if response['Version']['Driver Name'] == 'megaraid_sas':
                handle_megaraid_controller(response)
            elif response['Version']['Driver Name'] == 'mpt3sas':
                handle_sas_controller(response)
    except KeyError:
        pass

    print_all_metrics(metric_list)

def handle_common_controller(response):
    (controller_index, baselabel) = get_basic_controller_info(response)

    # Split up string to not trigger CodeSpell issues
    if 'ROC temperature(Degree Celc' + 'ius)' in response['HwCfg'].keys():
        response['HwCfg']['ROC temperature(Degree Celsius)'] = response['HwCfg'].pop('ROC temperature(Degree Celc' + 'ius)')
    add_metric('temperature', baselabel, int(response['HwCfg']['ROC temperature(Degree Celsius)']))

def handle_sas_controller(response):
    (controller_index, baselabel) = get_basic_controller_info(response)
    add_metric('healthy', baselabel, int(response['Status']['Controller Status'] == 'OK'))
    add_metric('ports', baselabel, response['HwCfg']['Backend Port Count'])
    try:
        # The number of physical disks is half of the number of items in this dict
        # Every disk is listed twice - once for basic info, again for detailed info
        add_metric('physical_drives', baselabel,
                   len(response['Physical Device Information'].keys()) / 2)
    except AttributeError:
        pass

    for key, basic_disk_info in response['Physical Device Information'].items():
        if 'Detailed Information' in key:
            continue
        create_metrcis_of_physical_drive(basic_disk_info[0],
                                         response['Physical Device Information'], controller_index)

def handle_megaraid_controller(response):
    (controller_index, baselabel) = get_basic_controller_info(response)

    # BBU Status Optimal value is 0 for cachevault and 32 for BBU
    add_metric('battery_backup_healthy', baselabel,
               int(response['Status']['BBU Status'] in [0, 32]))
    add_metric('degraded', baselabel, int(response['Status']['Controller Status'] == 'Degraded'))
    add_metric('failed', baselabel, int(response['Status']['Controller Status'] == 'Failed'))
    add_metric('healthy', baselabel, int(response['Status']['Controller Status'] == 'Optimal'))
    add_metric('drive_groups', baselabel, response['Drive Groups'])
    add_metric('virtual_drives', baselabel, response['Virtual Drives'])
    add_metric('physical_drives', baselabel, response['Physical Drives'])
    add_metric('ports', baselabel, response['HwCfg']['Backend Port Count'])
    add_metric('scheduled_patrol_read', baselabel,
               int('hrs' in response['Scheduled Tasks']['Patrol Read Reoccurrence']))
    for cvidx, cvinfo in enumerate(response['Cachevault_Info']):
        add_metric('cv_temperature', baselabel + ',cvidx="' + str(cvidx) + '"', int(cvinfo['Temp'].replace('C','')))

    time_difference_seconds = -1
    system_time = datetime.strptime(response['Basics'].get('Current System Date/time'),
                                    "%m/%d/%Y, %H:%M:%S")
    controller_time = datetime.strptime(response['Basics'].get('Current Controller Date/Time'),
                                        "%m/%d/%Y, %H:%M:%S")
    if system_time and controller_time:
        time_difference_seconds = abs(system_time - controller_time).seconds
        add_metric('time_difference', baselabel, time_difference_seconds)

    for virtual_drive in response['VD LIST']:
        vd_position = virtual_drive.get('DG/VD')
        drive_group, volume_group = -1, -1
        if vd_position:
            drive_group = vd_position.split('/')[0]
            volume_group = vd_position.split('/')[1]
        vd_baselabel = 'controller="{0}",DG="{1}",VG="{2}"'.format(controller_index, drive_group,
                                                                volume_group)
        vd_info_label = vd_baselabel + ',name="{0}",cache="{1}",type="{2}",state="{3}"'.format(
            str(virtual_drive.get('Name')).strip(),
            str(virtual_drive.get('Cache')).strip(),
            str(virtual_drive.get('TYPE')).strip(),
            str(virtual_drive.get('State')).strip())
        add_metric('vd_info', vd_info_label, 1)

    if response['Physical Drives'] > 0:
        data = get_storcli_json('/cALL/eALL/sALL show all J')
        drive_info = data['Controllers'][controller_index]['Response Data']
    for physical_drive in response['PD LIST']:
        create_metrcis_of_physical_drive(physical_drive, drive_info, controller_index)

def get_basic_controller_info(response):
    controller_index = response['Basics']['Controller']
    baselabel = 'controller="{0}"'.format(controller_index)

    controller_info_label = baselabel + ',model="{0}",serial="{1}",fwversion="{2}"'.format(
        str(response['Basics']['Model']).strip(),
        str(response['Basics']['Serial Number']).strip(),
        str(response['Version']['Firmware Version']).strip(),
    )
    add_metric('controller_info', controller_info_label, 1)

    return (controller_index, baselabel)

def create_metrcis_of_physical_drive(physical_drive, detailed_info_array, controller_index):
    enclosure = physical_drive.get('EID:Slt').split(':')[0]
    slot = physical_drive.get('EID:Slt').split(':')[1]

    pd_baselabel = 'controller="{0}",enclosure="{1}",slot="{2}"'.format(controller_index, enclosure,
                                                                     slot)
    pd_info_label = pd_baselabel + \
        ',disk_id="{0}",interface="{1}",media="{2}",model="{3}",DG="{4}",state="{5}"'.format(
            str(physical_drive.get('DID')).strip(),
            str(physical_drive.get('Intf')).strip(),
            str(physical_drive.get('Med')).strip(),
            str(physical_drive.get('Model')).strip(),
            str(physical_drive.get('DG')).strip(),
            str(physical_drive.get('State')).strip())

    drive_identifier = 'Drive /c' + str(controller_index) + '/e' + str(enclosure) + '/s' + str(
        slot)
    if enclosure == ' ':
        drive_identifier = 'Drive /c' + str(controller_index) + '/s' + str(slot)
    try:
        info = detailed_info_array[drive_identifier + ' - Detailed Information']
        state = info[drive_identifier + ' State']
        attributes = info[drive_identifier + ' Device attributes']
        settings = info[drive_identifier + ' Policies/Settings']

        add_metric('pd_shield_counter', pd_baselabel, state['Shield Counter'])
        add_metric('pd_media_errors', pd_baselabel, state['Media Error Count'])
        add_metric('pd_other_errors', pd_baselabel, state['Other Error Count'])
        add_metric('pd_predictive_errors', pd_baselabel, state['Predictive Failure Count'])
        add_metric('pd_smart_alerted', pd_baselabel,
                   int(state['S.M.A.R.T alert flagged by drive'] == 'Yes'))
        add_metric('pd_link_speed_gbps', pd_baselabel, attributes['Link Speed'].split('.')[0])
        add_metric('pd_device_speed_gbps', pd_baselabel, attributes['Device Speed'].split('.')[0])
        add_metric('pd_commissioned_spare', pd_baselabel,
                   int(settings['Commissioned Spare'] == 'Yes'))
        add_metric('pd_emergency_spare', pd_baselabel, int(settings['Emergency Spare'] == 'Yes'))
        pd_info_label += ',firmware="{0}"'.format(attributes['Firmware Revision'].strip())
    except KeyError:
        pass
    add_metric('pd_info', pd_info_label, 1)

def add_metric(name, labels, value):
    global metric_list
    try:
        metric_list[name].append({
            'labels': labels,
            'value': float(value),
        })
    except ValueError:
        pass

def print_all_metrics(metrics):
    for metric, measurements in metrics.items():
        print('# HELP {0}{1} MegaRAID {2}'.format(metric_prefix, metric, metric.replace('_', ' ')))
        print('# TYPE {0}{1} gauge'.format(metric_prefix, metric))
        for measurement in measurements:
            if measurement['value'] != 'Unknown':
                print('{0}{1}{2} {3}'.format(metric_prefix, metric, '{' + measurement['labels'] + '}',
                                         measurement['value']))

def get_storcli_json(storcli_args):
    """Get storcli output in JSON format."""
    # Check if storcli is installed and executable
    if not (os.path.isfile(storcli_path) and os.access(storcli_path, os.X_OK)):
        SystemExit(1)
    storcli_cmd = shlex.split(storcli_path + ' ' + storcli_args)
    proc = subprocess.Popen(
        storcli_cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output_json = proc.communicate()[0]
    data = json.loads(output_json.decode("utf-8"))

    if data["Controllers"][0]["Command Status"]["Status"] != "Success":
        SystemExit(1)
    return data

if __name__ == "__main__":
    PARSER = argparse.ArgumentParser(
        description=DESCRIPTION, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    PARSER.add_argument(
        '--storcli_path', default='/opt/MegaRAID/storcli/storcli64', help='path to StorCLi binary')
    PARSER.add_argument('--version', action='version', version='%(prog)s {0}'.format(VERSION))
    ARGS = PARSER.parse_args()

    main(ARGS)
<|endoftext|>


<commit_before>import numpy as np
import random
import tensorflow as tf

from collections import deque

class DiscreteDeepQ(object):
    def __init__(self, observation_size,
                       num_actions,
                       observation_to_actions,
                       optimizer,
                       session,
                       random_action_probability=0.05,
                       exploration_period=1000,
                       store_every_nth=5,
                       train_every_nth=5,
                       minibatch_size=32,
                       discount_rate=0.95,
                       max_experience=30000,
                       summary_writer=None):
        """Initialized the Deepq object.

        Based on:
            https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf

        Parameters
        -------
        observation_size : int
            length of the vector passed as observation
        num_actions : int
            number of actions that the model can execute
        observation_to_actions: dali model
            model that implements activate function
            that can take in observation vector or a batch
            and returns scores (of unbounded values) for each
            action for each observation.
            input shape:  [batch_size, observation_size]
            output shape: [batch_size, num_actions]
        optimizer: tf.solver.*
            optimizer for prediction error
        session: tf.Session
            session on which to execute the computation
        random_action_probability: float (0 to 1)
        exploration_period: int
            probability of choosing a random
            action (epsilon form paper) annealed linearly
            from 1 to random_action_probability over
            exploration_period
        store_every_nth: int
            to further decorrelate samples do not all
            transitions, but rather every nth transition.
            For example if store_every_nth is 5, then
            only 20% of all the transitions is stored.
        train_every_nth: int
            normally training_step is invoked every
            time action is executed. Depending on the
            setup that might be too often. When this
            variable is set set to n, then only every
            n-th time training_step is called will
            the training procedure actually be executed.
        minibatch_size: int
            number of state,action,reward,newstate
            tuples considered during experience reply
        dicount_rate: float (0 to 1)
            how much we care about future rewards.
        max_experience: int
            maximum size of the reply buffer
        summary_writer: tf.train.SummaryWriter
            writer to log metrics
        """
        # memorize arguments
        self.observation_size          = observation_size
        self.num_actions               = num_actions

        self.observation_to_actions    = observation_to_actions
        self.optimizer                 = optimizer
        self.s                         = session

        self.random_action_probability = random_action_probability
        self.exploration_period        = exploration_period
        self.store_every_nth           = store_every_nth
        self.train_every_nth           = train_every_nth
        self.minibatch_size            = minibatch_size
        self.discount_rate             = tf.constant(discount_rate)
        self.max_experience            = max_experience

        # deepq state
        self.actions_executed_so_far = 0
        self.experience = deque()

        self.iteration = 0
        self.summary_writer = summary_writer

        self.number_of_times_store_called = 0
        self.number_of_times_train_called = 0

        self.create_variables()

    def linear_annealing(self, n, total, p_initial, p_final):
        """Linear annealing between p_initial and p_final
        over total steps - computes value at step n"""
        if n >= total:
            return p_final
        else:
            return p_initial - (n * (p_initial - p_final)) / (total)

    def create_variables(self):
        # FOR REGULAR ACTION SCORE COMPUTATION
        with tf.name_scope("observation"):
            self.observation        = tf.placeholder(tf.float32, (None, self.observation_size), name="observation")
            self.action_scores      = self.observation_to_actions(self.observation)
            self.predicted_actions  = tf.argmax(self.action_scores, dimension=1, name="predicted_actions")

        with tf.name_scope("future_rewards"):
            # FOR PREDICTING TARGET FUTURE REWARDS
            self.observation_mask     = tf.placeholder(tf.float32, (None,), name="observation_mask")
            self.rewards              = tf.placeholder(tf.float32, (None,), name="rewards")
            target_values             = tf.reduce_max(self.action_scores, reduction_indices=[1,]) * self.observation_mask
            self.future_rewards       = self.rewards + self.discount_rate * target_values

        with tf.name_scope("q_value_precition"):
            # FOR PREDICTION ERROR
            self.action_mask                = tf.placeholder(tf.float32, (None, self.num_actions))
            self.masked_action_scores       = tf.reduce_sum(self.action_scores * self.action_mask, reduction_indices=[1,])
            self.precomputed_future_rewards = tf.placeholder(tf.float32, (None,))
            temp_diff                       = self.masked_action_scores - self.precomputed_future_rewards
            self.prediction_error           = tf.reduce_mean(tf.square(temp_diff))
            self.train_op                   = self.optimizer.minimize(self.prediction_error)

        self.metrics = [
            tf.scalar_summary("prediction_error", self.prediction_error)
        ]

    def action(self, observation):
        """Given observation returns the action that should be chosen using
        DeepQ learning strategy. Does not backprop."""
        assert len(observation.shape) == 1, \
                "Action is performed based on single observation."

        self.actions_executed_so_far += 1
        exploration_p = self.linear_annealing(self.actions_executed_so_far,
                                              self.exploration_period,
                                              1.0,
                                              self.random_action_probability)

        if random.random() < exploration_p:
            return random.randint(0, self.num_actions - 1)
        else:
            return self.s.run(self.predicted_actions, {self.observation: observation[np.newaxis,:]})[0]

    def store(self, observation, action, reward, newobservation):
        """Store experience, where starting with observation and
        execution action, we arrived at the newobservation and got the
        reward reward

        If newstate is None, the state/action pair is assumed to be terminal
        """
        if self.number_of_times_store_called % self.store_every_nth == 0:
            self.experience.append((observation, action, reward, newobservation))
            if len(self.experience) > self.max_experience:
                self.experience.popleft()
        self.number_of_times_store_called += 1

    def training_step(self):
        """Pick a self.minibatch_size exeperiences from reply buffer
        and backpropage the value function.
        """
        if self.number_of_times_train_called % self.train_every_nth == 0:
            if len(self.experience) <  self.minibatch_size:
                return

            # sample experience.
            samples   = random.sample(range(len(self.experience)), self.minibatch_size)
            samples   = [self.experience[i] for i in samples]

            # bach states
            states         = np.empty((len(samples), self.observation_size))
            newstates      = np.empty((len(samples), self.observation_size))
            action_mask    = np.zeros((len(samples), self.num_actions))

            newstates_mask = np.empty((len(samples),))
            rewards        = np.empty((len(samples),))

            for i, (state, action, reward, newstate) in enumerate(samples):
                states[i] = state
                action_mask[i] = 0
                action_mask[i][action] = 1
                rewards[i] = reward
                if newstate is not None:
                    newstates[i] = state
                    newstates_mask[i] = 1
                else:
                    newstates[i] = 0
                    newstates_mask[i] = 0

            future_rewards = self.s.run(self.future_rewards, {
                self.observation:      newstates,
                self.observation_mask: newstates_mask,
                self.rewards:          rewards,
            })

            res = self.s.run([self.prediction_error, self.train_op] + self.metrics, {
                self.observation:                states,
                self.action_mask:                action_mask,
                self.precomputed_future_rewards: future_rewards,
            })
            cost, metrics = res[0], res[2:]

            if self.summary_writer is not None:
                for metric in metrics:
                    self.summary_writer.add_summary(metric, self.iteration)
            self.iteration += 1

        self.number_of_times_train_called += 1
<commit_msg>Named more things<commit_after>import numpy as np
import random
import tensorflow as tf

from collections import deque

class DiscreteDeepQ(object):
    def __init__(self, observation_size,
                       num_actions,
                       observation_to_actions,
                       optimizer,
                       session,
                       random_action_probability=0.05,
                       exploration_period=1000,
                       store_every_nth=5,
                       train_every_nth=5,
                       minibatch_size=32,
                       discount_rate=0.95,
                       max_experience=30000,
                       summary_writer=None):
        """Initialized the Deepq object.

        Based on:
            https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf

        Parameters
        -------
        observation_size : int
            length of the vector passed as observation
        num_actions : int
            number of actions that the model can execute
        observation_to_actions: dali model
            model that implements activate function
            that can take in observation vector or a batch
            and returns scores (of unbounded values) for each
            action for each observation.
            input shape:  [batch_size, observation_size]
            output shape: [batch_size, num_actions]
        optimizer: tf.solver.*
            optimizer for prediction error
        session: tf.Session
            session on which to execute the computation
        random_action_probability: float (0 to 1)
        exploration_period: int
            probability of choosing a random
            action (epsilon form paper) annealed linearly
            from 1 to random_action_probability over
            exploration_period
        store_every_nth: int
            to further decorrelate samples do not all
            transitions, but rather every nth transition.
            For example if store_every_nth is 5, then
            only 20% of all the transitions is stored.
        train_every_nth: int
            normally training_step is invoked every
            time action is executed. Depending on the
            setup that might be too often. When this
            variable is set set to n, then only every
            n-th time training_step is called will
            the training procedure actually be executed.
        minibatch_size: int
            number of state,action,reward,newstate
            tuples considered during experience reply
        dicount_rate: float (0 to 1)
            how much we care about future rewards.
        max_experience: int
            maximum size of the reply buffer
        summary_writer: tf.train.SummaryWriter
            writer to log metrics
        """
        # memorize arguments
        self.observation_size          = observation_size
        self.num_actions               = num_actions

        self.observation_to_actions    = observation_to_actions
        self.optimizer                 = optimizer
        self.s                         = session

        self.random_action_probability = random_action_probability
        self.exploration_period        = exploration_period
        self.store_every_nth           = store_every_nth
        self.train_every_nth           = train_every_nth
        self.minibatch_size            = minibatch_size
        self.discount_rate             = tf.constant(discount_rate)
        self.max_experience            = max_experience

        # deepq state
        self.actions_executed_so_far = 0
        self.experience = deque()

        self.iteration = 0
        self.summary_writer = summary_writer

        self.number_of_times_store_called = 0
        self.number_of_times_train_called = 0

        self.create_variables()

    def linear_annealing(self, n, total, p_initial, p_final):
        """Linear annealing between p_initial and p_final
        over total steps - computes value at step n"""
        if n >= total:
            return p_final
        else:
            return p_initial - (n * (p_initial - p_final)) / (total)

    def create_variables(self):
        # FOR REGULAR ACTION SCORE COMPUTATION
        with tf.name_scope("observation"):
            self.observation        = tf.placeholder(tf.float32, (None, self.observation_size), name="observation")
            self.action_scores      = self.observation_to_actions(self.observation)
            self.predicted_actions  = tf.argmax(self.action_scores, dimension=1, name="predicted_actions")

        with tf.name_scope("future_rewards"):
            # FOR PREDICTING TARGET FUTURE REWARDS
            self.observation_mask     = tf.placeholder(tf.float32, (None,), name="observation_mask")
            self.rewards              = tf.placeholder(tf.float32, (None,), name="rewards")
            target_values             = tf.reduce_max(self.action_scores, reduction_indices=[1,]) * self.observation_mask
            self.future_rewards       = self.rewards + self.discount_rate * target_values

        with tf.name_scope("q_value_precition"):
            # FOR PREDICTION ERROR
            self.action_mask                = tf.placeholder(tf.float32, (None, self.num_actions), name="action_mask")
            self.masked_action_scores       = tf.reduce_sum(self.action_scores * self.action_mask, reduction_indices=[1,])
            self.precomputed_future_rewards = tf.placeholder(tf.float32, (None,), "precomputed_rewards")
            temp_diff                       = self.masked_action_scores - self.precomputed_future_rewards
            self.prediction_error           = tf.reduce_mean(tf.square(temp_diff))
            self.train_op                   = self.optimizer.minimize(self.prediction_error)

        self.metrics = [
            tf.scalar_summary("prediction_error", self.prediction_error)
        ]

    def action(self, observation):
        """Given observation returns the action that should be chosen using
        DeepQ learning strategy. Does not backprop."""
        assert len(observation.shape) == 1, \
                "Action is performed based on single observation."

        self.actions_executed_so_far += 1
        exploration_p = self.linear_annealing(self.actions_executed_so_far,
                                              self.exploration_period,
                                              1.0,
                                              self.random_action_probability)

        if random.random() < exploration_p:
            return random.randint(0, self.num_actions - 1)
        else:
            return self.s.run(self.predicted_actions, {self.observation: observation[np.newaxis,:]})[0]

    def store(self, observation, action, reward, newobservation):
        """Store experience, where starting with observation and
        execution action, we arrived at the newobservation and got the
        reward reward

        If newstate is None, the state/action pair is assumed to be terminal
        """
        if self.number_of_times_store_called % self.store_every_nth == 0:
            self.experience.append((observation, action, reward, newobservation))
            if len(self.experience) > self.max_experience:
                self.experience.popleft()
        self.number_of_times_store_called += 1

    def training_step(self):
        """Pick a self.minibatch_size exeperiences from reply buffer
        and backpropage the value function.
        """
        if self.number_of_times_train_called % self.train_every_nth == 0:
            if len(self.experience) <  self.minibatch_size:
                return

            # sample experience.
            samples   = random.sample(range(len(self.experience)), self.minibatch_size)
            samples   = [self.experience[i] for i in samples]

            # bach states
            states         = np.empty((len(samples), self.observation_size))
            newstates      = np.empty((len(samples), self.observation_size))
            action_mask    = np.zeros((len(samples), self.num_actions))

            newstates_mask = np.empty((len(samples),))
            rewards        = np.empty((len(samples),))

            for i, (state, action, reward, newstate) in enumerate(samples):
                states[i] = state
                action_mask[i] = 0
                action_mask[i][action] = 1
                rewards[i] = reward
                if newstate is not None:
                    newstates[i] = state
                    newstates_mask[i] = 1
                else:
                    newstates[i] = 0
                    newstates_mask[i] = 0

            future_rewards = self.s.run(self.future_rewards, {
                self.observation:      newstates,
                self.observation_mask: newstates_mask,
                self.rewards:          rewards,
            })

            res = self.s.run([self.prediction_error, self.train_op] + self.metrics, {
                self.observation:                states,
                self.action_mask:                action_mask,
                self.precomputed_future_rewards: future_rewards,
            })
            cost, metrics = res[0], res[2:]

            if self.summary_writer is not None:
                for metric in metrics:
                    self.summary_writer.add_summary(metric, self.iteration)
            self.iteration += 1

        self.number_of_times_train_called += 1
<|endoftext|>