<commit_before># Copyright 2016 Grist Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ast
import collections
import token
import sys
from six import iteritems, PY3
from abc import ABCMeta
from typing import Callable, Dict, Iterator, List, Optional, Tuple, Union, cast, Any
from ast import Module, expr, AST
from astroid.node_classes import NodeNG # type: ignore[import]

def token_repr(tok_type, string):
  # type: (int, Optional[str]) -> str
  """Returns a human-friendly representation of a token with the given type and string."""
  # repr() prefixes unicode with 'u' on Python2 but not Python3; strip it out for consistency.
  return '%s:%s' % (token.tok_name[tok_type], repr(string).lstrip('u'))

class Token(collections.namedtuple('Token', 'type string start end line index startpos endpos')):
  """
  TokenInfo is an 8-tuple containing the same 5 fields as the tokens produced by the tokenize
  module, and 3 additional ones useful for this module:

  - [0] .type     Token type (see token.py)
  - [1] .string   Token (a string)
  - [2] .start    Starting (row, column) indices of the token (a 2-tuple of ints)
  - [3] .end      Ending (row, column) indices of the token (a 2-tuple of ints)
  - [4] .line     Original line (string)
  - [5] .index    Index of the token in the list of tokens that it belongs to.
  - [6] .startpos Starting character offset into the input text.
  - [7] .endpos   Ending character offset into the input text.
  """
  def __str__(self):
    # type: () -> str
    return token_repr(self.type, self.string)

# Type class used to expand out the definition of AST to include fields added by this library
# It's not actually used for anything other than type checking though!
class EnhancedAST(AST):
  # Additional attributes set by mark_tokens
  first_token = None # type: Token
  last_token = None # type: Token
  lineno = 0 # type: int

if PY3:
  AstConstant = ast.Constant
else:
  class AstConstant:
    value = object()

AstNode = Union[EnhancedAST, NodeNG]

def match_token(token, tok_type, tok_str=None):
  # type: (Token, int, Optional[str]) -> bool
  """Returns true if token is of the given type and, if a string is given, has that string."""
  return token.type == tok_type and (tok_str is None or token.string == tok_str)

def expect_token(token, tok_type, tok_str=None):
  # type: (Token, int, Optional[str]) -> None
  """
  Verifies that the given token is of the expected type. If tok_str is given, the token string
  is verified too. If the token doesn't match, raises an informative ValueError.
  """
  if not match_token(token, tok_type, tok_str):
    raise ValueError("Expected token %s, got %s on line %s col %s" % (
      token_repr(tok_type, tok_str), str(token),
      token.start[0], token.start[1] + 1))

# These were previously defined in tokenize.py and distinguishable by being greater than
# token.N_TOKEN. As of python3.7, they are in token.py, and we check for them explicitly.
if sys.version_info >= (3, 7):
  def is_non_coding_token(token_type):
    # type: (int) -> bool
    """
    These are considered non-coding tokens, as they don't affect the syntax tree.
    """
    return token_type in (token.NL, token.COMMENT, token.ENCODING)
else:
  def is_non_coding_token(token_type):
    # type: (int) -> bool
    """
    These are considered non-coding tokens, as they don't affect the syntax tree.
    """
    return token_type >= token.N_TOKENS

def iter_children_func(node):
  # type: (Module) -> Callable
  """
  Returns a function which yields all direct children of a AST node,
  skipping children that are singleton nodes.
  The function depends on whether ``node`` is from ``ast`` or from the ``astroid`` module.
  """
  return iter_children_astroid if hasattr(node, 'get_children') else iter_children_ast

def iter_children_astroid(node):
  # type: (NodeNG) -> Union[Iterator, List]
  # Don't attempt to process children of JoinedStr nodes, which we can't fully handle yet.
  if is_joined_str(node):
    return []

  return node.get_children()

SINGLETONS = {c for n, c in iteritems(ast.__dict__) if isinstance(c, type) and
              issubclass(c, (ast.expr_context, ast.boolop, ast.operator, ast.unaryop, ast.cmpop))}

def iter_children_ast(node):
  # type: (AST) -> Iterator[Union[AST, expr]]
  # Don't attempt to process children of JoinedStr nodes, which we can't fully handle yet.
  if is_joined_str(node):
    return

  if isinstance(node, ast.Dict):
    # override the iteration order: instead of <all keys>, <all values>,
    # yield keys and values in source order (key1, value1, key2, value2, ...)
    for (key, value) in zip(node.keys, node.values):
      if key is not None:
        yield key
      yield value
    return

  for child in ast.iter_child_nodes(node):
    # Skip singleton children; they don't reflect particular positions in the code and break the
    # assumptions about the tree consisting of distinct nodes. Note that collecting classes
    # beforehand and checking them in a set is faster than using isinstance each time.
    if child.__class__ not in SINGLETONS:
      yield child

stmt_class_names = {n for n, c in iteritems(ast.__dict__)
                    if isinstance(c, type) and issubclass(c, ast.stmt)}
expr_class_names = ({n for n, c in iteritems(ast.__dict__)
                    if isinstance(c, type) and issubclass(c, ast.expr)} |
                    {'AssignName', 'DelName', 'Const', 'AssignAttr', 'DelAttr'})

# These feel hacky compared to isinstance() but allow us to work with both ast and astroid nodes
# in the same way, and without even importing astroid.
def is_expr(node):
  # type: (AstNode) -> bool
  """Returns whether node is an expression node."""
  return node.__class__.__name__ in expr_class_names

def is_stmt(node):
  # type: (AstNode) -> bool
  """Returns whether node is a statement node."""
  return node.__class__.__name__ in stmt_class_names

def is_module(node):
  # type: (AstNode) -> bool
  """Returns whether node is a module node."""
  return node.__class__.__name__ == 'Module'

def is_joined_str(node):
  # type: (AstNode) -> bool
  """Returns whether node is a JoinedStr node, used to represent f-strings."""
  # At the moment, nodes below JoinedStr have wrong line/col info, and trying to process them only
  # leads to errors.
  return node.__class__.__name__ == 'JoinedStr'

def is_starred(node):
  # type: (AstNode) -> bool
  """Returns whether node is a starred expression node."""
  return node.__class__.__name__ == 'Starred'

def is_slice(node):
  # type: (AstNode) -> bool
  """Returns whether node represents a slice, e.g. `1:2` in `x[1:2]`"""
  # Before 3.9, a tuple containing a slice is an ExtSlice,
  # but this was removed in https://bugs.python.org/issue34822
  return (
      node.__class__.__name__ in ('Slice', 'ExtSlice')
      or (
          node.__class__.__name__ == 'Tuple'
          and any(map(is_slice, cast(ast.Tuple, node).elts))
      )
  )

# Sentinel value used by visit_tree().
_PREVISIT = object()

def visit_tree(node, previsit, postvisit):
  # type: (Module, Callable[[AstNode, Optional[Token]], Tuple[Optional[Token], Optional[Token]]], Optional[Callable[[AstNode, Optional[Token], Optional[Token]], None]])   -> None
  """
  Scans the tree under the node depth-first using an explicit stack. It avoids implicit recursion
  via the function call stack to avoid hitting 'maximum recursion depth exceeded' error.

  It calls ``previsit()`` and ``postvisit()`` as follows:

  * ``previsit(node, par_value)`` - should return ``(par_value, value)``
        ``par_value`` is as returned from ``previsit()`` of the parent.

  * ``postvisit(node, par_value, value)`` - should return ``value``
        ``par_value`` is as returned from ``previsit()`` of the parent, and ``value`` is as
        returned from ``previsit()`` of this node itself. The return ``value`` is ignored except
        the one for the root node, which is returned from the overall ``visit_tree()`` call.

  For the initial node, ``par_value`` is None. ``postvisit`` may be None.
  """
  if not postvisit:
    postvisit = lambda node, pvalue, value: None

  iter_children = iter_children_func(node)
  done = set()
  ret = None
  stack = [(node, None, _PREVISIT)] # type: List[Tuple[AstNode, Optional[Token], Union[Optional[Token], object]]]
  while stack:
    current, par_value, value = stack.pop()
    if value is _PREVISIT:
      assert current not in done    # protect againt infinite loop in case of a bad tree.
      done.add(current)

      pvalue, post_value = previsit(current, par_value)
      stack.append((current, par_value, post_value))

      # Insert all children in reverse order (so that first child ends up on top of the stack).
      ins = len(stack)
      for n in iter_children(current):
        stack.insert(ins, (n, pvalue, _PREVISIT))
    else:
      ret = postvisit(current, par_value, cast(Optional[Token], value))
  return ret

def walk(node):
  # type: (Module) -> Iterator[Union[Module, AstNode]]
  """
  Recursively yield all descendant nodes in the tree starting at ``node`` (including ``node``
  itself), using depth-first pre-order traversal (yieling parents before their children).

  This is similar to ``ast.walk()``, but with a different order, and it works for both ``ast`` and
  ``astroid`` trees. Also, as ``iter_children()``, it skips singleton nodes generated by ``ast``.
  """
  iter_children = iter_children_func(node)
  done = set()
  stack = [node]
  while stack:
    current = stack.pop()
    assert current not in done    # protect againt infinite loop in case of a bad tree.
    done.add(current)

    yield current

    # Insert all children in reverse order (so that first child ends up on top of the stack).
    # This is faster than building a list and reversing it.
    ins = len(stack)
    for c in iter_children(current):
      stack.insert(ins, c)

def replace(text, replacements):
  # type: (str, List[Tuple[int, int, str]]) -> str
  """
  Replaces multiple slices of text with new values. This is a convenience method for making code
  modifications of ranges e.g. as identified by ``ASTTokens.get_text_range(node)``. Replacements is
  an iterable of ``(start, end, new_text)`` tuples.

  For example, ``replace("this is a test", [(0, 4, "X"), (8, 9, "THE")])`` produces
  ``"X is THE test"``.
  """
  p = 0
  parts = []
  for (start, end, new_text) in sorted(replacements):
    parts.append(text[p:start])
    parts.append(new_text)
    p = end
  parts.append(text[p:])
  return ''.join(parts)

class NodeMethods(object):
  """
  Helper to get `visit_{node_type}` methods given a node's class and cache the results.
  """
  def __init__(self):
    # type: () -> None
    self._cache = {} # type: Dict[Union[ABCMeta, type], Callable[[AstNode, Token, Token], Tuple[Token, Token]]]

  def get(self, obj, cls):
    # type: (Any, Union[ABCMeta, type]) -> Callable
    """
    Using the lowercase name of the class as node_type, returns `obj.visit_{node_type}`,
    or `obj.visit_default` if the type-specific method is not found.
    """
    method = self._cache.get(cls)
    if not method:
      name = "visit_" + cls.__name__.lower()
      method = getattr(obj, name, obj.visit_default)
      self._cache[cls] = method
    return method
<commit_msg>Fake constant also needed for 3.5<commit_after># Copyright 2016 Grist Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ast
import collections
import token
import sys
from six import iteritems
from abc import ABCMeta
from typing import Callable, Dict, Iterator, List, Optional, Tuple, Union, cast, Any
from ast import Module, expr, AST
from astroid.node_classes import NodeNG # type: ignore[import]

def token_repr(tok_type, string):
  # type: (int, Optional[str]) -> str
  """Returns a human-friendly representation of a token with the given type and string."""
  # repr() prefixes unicode with 'u' on Python2 but not Python3; strip it out for consistency.
  return '%s:%s' % (token.tok_name[tok_type], repr(string).lstrip('u'))

class Token(collections.namedtuple('Token', 'type string start end line index startpos endpos')):
  """
  TokenInfo is an 8-tuple containing the same 5 fields as the tokens produced by the tokenize
  module, and 3 additional ones useful for this module:

  - [0] .type     Token type (see token.py)
  - [1] .string   Token (a string)
  - [2] .start    Starting (row, column) indices of the token (a 2-tuple of ints)
  - [3] .end      Ending (row, column) indices of the token (a 2-tuple of ints)
  - [4] .line     Original line (string)
  - [5] .index    Index of the token in the list of tokens that it belongs to.
  - [6] .startpos Starting character offset into the input text.
  - [7] .endpos   Ending character offset into the input text.
  """
  def __str__(self):
    # type: () -> str
    return token_repr(self.type, self.string)

# Type class used to expand out the definition of AST to include fields added by this library
# It's not actually used for anything other than type checking though!
class EnhancedAST(AST):
  # Additional attributes set by mark_tokens
  first_token = None # type: Token
  last_token = None # type: Token
  lineno = 0 # type: int

if sys.version_info >= (3, 6):
  AstConstant = ast.Constant
else:
  class AstConstant:
    value = object()

AstNode = Union[EnhancedAST, NodeNG]

def match_token(token, tok_type, tok_str=None):
  # type: (Token, int, Optional[str]) -> bool
  """Returns true if token is of the given type and, if a string is given, has that string."""
  return token.type == tok_type and (tok_str is None or token.string == tok_str)

def expect_token(token, tok_type, tok_str=None):
  # type: (Token, int, Optional[str]) -> None
  """
  Verifies that the given token is of the expected type. If tok_str is given, the token string
  is verified too. If the token doesn't match, raises an informative ValueError.
  """
  if not match_token(token, tok_type, tok_str):
    raise ValueError("Expected token %s, got %s on line %s col %s" % (
      token_repr(tok_type, tok_str), str(token),
      token.start[0], token.start[1] + 1))

# These were previously defined in tokenize.py and distinguishable by being greater than
# token.N_TOKEN. As of python3.7, they are in token.py, and we check for them explicitly.
if sys.version_info >= (3, 7):
  def is_non_coding_token(token_type):
    # type: (int) -> bool
    """
    These are considered non-coding tokens, as they don't affect the syntax tree.
    """
    return token_type in (token.NL, token.COMMENT, token.ENCODING)
else:
  def is_non_coding_token(token_type):
    # type: (int) -> bool
    """
    These are considered non-coding tokens, as they don't affect the syntax tree.
    """
    return token_type >= token.N_TOKENS

def iter_children_func(node):
  # type: (Module) -> Callable
  """
  Returns a function which yields all direct children of a AST node,
  skipping children that are singleton nodes.
  The function depends on whether ``node`` is from ``ast`` or from the ``astroid`` module.
  """
  return iter_children_astroid if hasattr(node, 'get_children') else iter_children_ast

def iter_children_astroid(node):
  # type: (NodeNG) -> Union[Iterator, List]
  # Don't attempt to process children of JoinedStr nodes, which we can't fully handle yet.
  if is_joined_str(node):
    return []

  return node.get_children()

SINGLETONS = {c for n, c in iteritems(ast.__dict__) if isinstance(c, type) and
              issubclass(c, (ast.expr_context, ast.boolop, ast.operator, ast.unaryop, ast.cmpop))}

def iter_children_ast(node):
  # type: (AST) -> Iterator[Union[AST, expr]]
  # Don't attempt to process children of JoinedStr nodes, which we can't fully handle yet.
  if is_joined_str(node):
    return

  if isinstance(node, ast.Dict):
    # override the iteration order: instead of <all keys>, <all values>,
    # yield keys and values in source order (key1, value1, key2, value2, ...)
    for (key, value) in zip(node.keys, node.values):
      if key is not None:
        yield key
      yield value
    return

  for child in ast.iter_child_nodes(node):
    # Skip singleton children; they don't reflect particular positions in the code and break the
    # assumptions about the tree consisting of distinct nodes. Note that collecting classes
    # beforehand and checking them in a set is faster than using isinstance each time.
    if child.__class__ not in SINGLETONS:
      yield child

stmt_class_names = {n for n, c in iteritems(ast.__dict__)
                    if isinstance(c, type) and issubclass(c, ast.stmt)}
expr_class_names = ({n for n, c in iteritems(ast.__dict__)
                    if isinstance(c, type) and issubclass(c, ast.expr)} |
                    {'AssignName', 'DelName', 'Const', 'AssignAttr', 'DelAttr'})

# These feel hacky compared to isinstance() but allow us to work with both ast and astroid nodes
# in the same way, and without even importing astroid.
def is_expr(node):
  # type: (AstNode) -> bool
  """Returns whether node is an expression node."""
  return node.__class__.__name__ in expr_class_names

def is_stmt(node):
  # type: (AstNode) -> bool
  """Returns whether node is a statement node."""
  return node.__class__.__name__ in stmt_class_names

def is_module(node):
  # type: (AstNode) -> bool
  """Returns whether node is a module node."""
  return node.__class__.__name__ == 'Module'

def is_joined_str(node):
  # type: (AstNode) -> bool
  """Returns whether node is a JoinedStr node, used to represent f-strings."""
  # At the moment, nodes below JoinedStr have wrong line/col info, and trying to process them only
  # leads to errors.
  return node.__class__.__name__ == 'JoinedStr'

def is_starred(node):
  # type: (AstNode) -> bool
  """Returns whether node is a starred expression node."""
  return node.__class__.__name__ == 'Starred'

def is_slice(node):
  # type: (AstNode) -> bool
  """Returns whether node represents a slice, e.g. `1:2` in `x[1:2]`"""
  # Before 3.9, a tuple containing a slice is an ExtSlice,
  # but this was removed in https://bugs.python.org/issue34822
  return (
      node.__class__.__name__ in ('Slice', 'ExtSlice')
      or (
          node.__class__.__name__ == 'Tuple'
          and any(map(is_slice, cast(ast.Tuple, node).elts))
      )
  )

# Sentinel value used by visit_tree().
_PREVISIT = object()

def visit_tree(node, previsit, postvisit):
  # type: (Module, Callable[[AstNode, Optional[Token]], Tuple[Optional[Token], Optional[Token]]], Optional[Callable[[AstNode, Optional[Token], Optional[Token]], None]])   -> None
  """
  Scans the tree under the node depth-first using an explicit stack. It avoids implicit recursion
  via the function call stack to avoid hitting 'maximum recursion depth exceeded' error.

  It calls ``previsit()`` and ``postvisit()`` as follows:

  * ``previsit(node, par_value)`` - should return ``(par_value, value)``
        ``par_value`` is as returned from ``previsit()`` of the parent.

  * ``postvisit(node, par_value, value)`` - should return ``value``
        ``par_value`` is as returned from ``previsit()`` of the parent, and ``value`` is as
        returned from ``previsit()`` of this node itself. The return ``value`` is ignored except
        the one for the root node, which is returned from the overall ``visit_tree()`` call.

  For the initial node, ``par_value`` is None. ``postvisit`` may be None.
  """
  if not postvisit:
    postvisit = lambda node, pvalue, value: None

  iter_children = iter_children_func(node)
  done = set()
  ret = None
  stack = [(node, None, _PREVISIT)] # type: List[Tuple[AstNode, Optional[Token], Union[Optional[Token], object]]]
  while stack:
    current, par_value, value = stack.pop()
    if value is _PREVISIT:
      assert current not in done    # protect againt infinite loop in case of a bad tree.
      done.add(current)

      pvalue, post_value = previsit(current, par_value)
      stack.append((current, par_value, post_value))

      # Insert all children in reverse order (so that first child ends up on top of the stack).
      ins = len(stack)
      for n in iter_children(current):
        stack.insert(ins, (n, pvalue, _PREVISIT))
    else:
      ret = postvisit(current, par_value, cast(Optional[Token], value))
  return ret

def walk(node):
  # type: (Module) -> Iterator[Union[Module, AstNode]]
  """
  Recursively yield all descendant nodes in the tree starting at ``node`` (including ``node``
  itself), using depth-first pre-order traversal (yieling parents before their children).

  This is similar to ``ast.walk()``, but with a different order, and it works for both ``ast`` and
  ``astroid`` trees. Also, as ``iter_children()``, it skips singleton nodes generated by ``ast``.
  """
  iter_children = iter_children_func(node)
  done = set()
  stack = [node]
  while stack:
    current = stack.pop()
    assert current not in done    # protect againt infinite loop in case of a bad tree.
    done.add(current)

    yield current

    # Insert all children in reverse order (so that first child ends up on top of the stack).
    # This is faster than building a list and reversing it.
    ins = len(stack)
    for c in iter_children(current):
      stack.insert(ins, c)

def replace(text, replacements):
  # type: (str, List[Tuple[int, int, str]]) -> str
  """
  Replaces multiple slices of text with new values. This is a convenience method for making code
  modifications of ranges e.g. as identified by ``ASTTokens.get_text_range(node)``. Replacements is
  an iterable of ``(start, end, new_text)`` tuples.

  For example, ``replace("this is a test", [(0, 4, "X"), (8, 9, "THE")])`` produces
  ``"X is THE test"``.
  """
  p = 0
  parts = []
  for (start, end, new_text) in sorted(replacements):
    parts.append(text[p:start])
    parts.append(new_text)
    p = end
  parts.append(text[p:])
  return ''.join(parts)

class NodeMethods(object):
  """
  Helper to get `visit_{node_type}` methods given a node's class and cache the results.
  """
  def __init__(self):
    # type: () -> None
    self._cache = {} # type: Dict[Union[ABCMeta, type], Callable[[AstNode, Token, Token], Tuple[Token, Token]]]

  def get(self, obj, cls):
    # type: (Any, Union[ABCMeta, type]) -> Callable
    """
    Using the lowercase name of the class as node_type, returns `obj.visit_{node_type}`,
    or `obj.visit_default` if the type-specific method is not found.
    """
    method = self._cache.get(cls)
    if not method:
      name = "visit_" + cls.__name__.lower()
      method = getattr(obj, name, obj.visit_default)
      self._cache[cls] = method
    return method
<|endoftext|>


<commit_before>
# LOAD CHIMP COMPONENTS

# Memory
from chimp.memories import ReplayMemoryHDF5

# Learner (Brain)
from chimp.learners.dqn_learner import DQNLearner
from chimp.learners.chainer_backend import ChainerBackend

# Simulator
from chimp.simulators.atari import AtariSimulator

# Agent Framework
from chimp.agents import DQNAgent

# Request necessary packages

import numpy as np
import random

import chainer
import chainer.functions as F
import chainer.links as L
from chainer import Chain

# Set the settings

settings = {

    # agent settings
    'batch_size' : 32,
    'print_every' : 1000,
    'save_dir' : './results_atari',
    'iterations' : 10000,
    'eval_iterations' : 5000,
    'eval_every' : 50000,
    'save_every' : 50000,
    'initial_exploration' : 50000,
    'epsilon_decay' : 0.000005, # subtract from epsilon every step
    'eval_epsilon' : 0.05, # epsilon used in evaluation, 0 means no random actions
    'epsilon' : 1.0,  # Initial exploratoin rate
    'learn_freq' : 4,
    'history_sizes' : (4, 0, 0), # sizes of histories to use as nn inputs (o, a, r)
    'model_dims' : (84,84),

    # Atari settings
    'rom' : "Breakout.bin",
    'rom_dir' :  './roms',
    'pad' : 15, # padding parameter - for image cropping - only along the length of the image, to obtain a square
    'action_history' : True,

    # simulator settings
    'viz' : True,
    'viz_cropped' : False,

    # replay memory settings
    'memory_size' : 1000000,  # size of replay memory
    'frame_skip' : 4,  # number of frames to skip

    # learner settings
    'learning_rate' : 0.00025,
    'decay_rate' : 0.95, # decay rate for RMSprop, otherwise not used
    'discount' : 0.99, # discount rate for RL
    'clip_err' : False, # value to clip loss gradients to
    'clip_reward' : 1, # value to clip reward values to
    'target_net_update' : 10000, # update the update-generating target net every fixed number of iterations
    'optim_name' : 'RMSprop', # currently supports "RMSprop", "ADADELTA", "ADAM" and "SGD"'
    'gpu' : True,
    'reward_rescale': False,

    # general
    'seed_general' : 1723,
    'seed_simulator' : 5632,
    'seed_agent' : 9826,
    'seed_memory' : 7563

    }

# Initialize random seed

np.random.seed(settings["seed_general"])
random.seed(settings["seed_general"])

# Initialize Atari simulator
simulator = AtariSimulator(settings)

# Initialize a deep net structure

class Convolution(Chain):

    def __init__(self):
        super(Convolution, self).__init__(
            l1=F.Convolution2D(settings['history_sizes'][0], 32, ksize=8, stride=4, nobias=False, wscale=np.sqrt(2)),
            l2=F.Convolution2D(32, 64, ksize=4, stride=2, nobias=False, wscale=np.sqrt(2)),
            l3=F.Convolution2D(64, 64, ksize=3, stride=1, nobias=False, wscale=np.sqrt(2)),
            l4=F.Linear(3136, 512, wscale = np.sqrt(2)),
            l5=F.Linear(512, simulator.n_actions, wscale = np.sqrt(2)),
        )

    def __call__(self, ohist, ahist):
        if len(ohist.data.shape) < 4:
            ohist = F.reshape(ohist,(1,4,84,84))
        h1 = F.relu(self.l1(ohist/255.0))
        h2 = F.relu(self.l2(h1))
        h3 = F.relu(self.l3(h2))
        h4 = F.relu(self.l4(h3))
        output = self.l5(h4)
        return output

net = Convolution()

# Initialize Learner with a Chainer backend

backend = ChainerBackend(settings)
backend.set_net(net)

learner = DQNLearner(settings, backend)

# Initialize memory

memory = ReplayMemoryHDF5(settings)

# Initialize Agent Framework

agent = DQNAgent(learner, memory, simulator, settings)

# Start training

agent.train()

# load net

# agent.learner.load_net(settings['save_dir']+'/net_750000.p')

# '''Plot training history'''

# plt.plot(range(len(learner.val_rewards)), learner.val_rewards)
# plt.xlabel("Evaluation episode (1000 transitions long, every 5000 training iter.)")
# plt.ylabel("Accumulated reward per episode")
# plt.xlim(0, 100)
# plt.ylim(-200, 200)
# plt.grid(True)
# plt.savefig(settings['save_dir'] + '_' + "evaluation_reward.svg", bbox_inches='tight')
# plt.close()

# plt.plot(range(len(learner.val_qval_avgs)), learner.val_qval_avgs)
# plt.xlabel("Evaluation episode  (1000 transitions long, every 5000 training iter.)")
# plt.ylabel("Average Q-value")
# plt.xlim(0, 100)
# plt.ylim(0, 20)
# plt.grid(True)
# plt.savefig(settings['save_dir'] + '_' + "evaluation_q_value.svg", bbox_inches='tight')
# plt.close()

# '''Plot value surface for beliefs'''
# ind_max = learner.val_rewards[0::5].index(max(learner.val_rewards[0::5]))
# ind_net = settings['save_every'] + ind_max * settings['save_every']
# learner.load_net(settings['save_dir']+'/net_%d.p' % int(ind_net))

# print(ind_net)
# print(ind_max*5)
# print(learner.val_rewards)
# print(learner.val_rewards[ind_max*5])

# '''Evaluate learned policy against the optimal heuristic policy'''
# np.random.seed(settings["seed_general"])

# print('Evaluating DQN agent...')
# print('(reward, MSE loss, mean Q-value, episodes - NA, time)')
# reward, MSE_loss, mean_Q_value, episodes, time, paths, actions, rewards = agent.evaluate(learner, simulator, 50000)
# print(reward, MSE_loss, mean_Q_value, episodes, time)
<commit_msg>Cleaned up run script for atari<commit_after>
# LOAD CHIMP COMPONENTS

# Memory
from chimp.memories import ReplayMemoryHDF5

# Learner (Brain)
from chimp.learners.dqn_learner import DQNLearner
from chimp.learners.chainer_backend import ChainerBackend

# Simulator
from chimp.simulators.atari import AtariSimulator

# Agent Framework
from chimp.agents import DQNAgent

# Request necessary packages

import numpy as np
import random

import chainer
import chainer.functions as F
import chainer.links as L
from chainer import Chain

# Set the settings

settings = {

    # agent settings
    'batch_size' : 32,
    'print_every' : 5000,
    'save_dir' : './results_atari',
    'iterations' : 5000000,
    'eval_iterations' : 5000,
    'eval_every' : 50000,
    'save_every' : 50000,
    'initial_exploration' : 50000,
    'epsilon_decay' : 0.000005, # subtract from epsilon every step
    'eval_epsilon' : 0.05, # epsilon used in evaluation, 0 means no random actions
    'epsilon' : 1.0,  # Initial exploratoin rate
    'learn_freq' : 4,
    'history_sizes' : (4, 0, 0), # sizes of histories to use as nn inputs (o, a, r)
    'model_dims' : (84,84),

    # Atari settings
    'rom' : "Breakout.bin",
    'rom_dir' :  './roms',
    'pad' : 15, # padding parameter - for image cropping - only along the length of the image, to obtain a square
    'action_history' : True,

    # simulator settings
    'viz' : False,
    'viz_cropped' : False,

    # replay memory settings
    'memory_size' : 1000000,  # size of replay memory
    'frame_skip' : 4,  # number of frames to skip

    # learner settings
    'learning_rate' : 0.00025,
    'decay_rate' : 0.95, # decay rate for RMSprop, otherwise not used
    'discount' : 0.99, # discount rate for RL
    'clip_err' : False, # value to clip loss gradients to
    'clip_reward' : 1, # value to clip reward values to
    'target_net_update' : 10000, # update the update-generating target net every fixed number of iterations
    'optim_name' : 'RMSprop', # currently supports "RMSprop", "ADADELTA", "ADAM" and "SGD"'
    'gpu' : True,
    'reward_rescale': False,

    # general
    'seed_general' : 1723,
    'seed_simulator' : 5632,
    'seed_agent' : 9826,
    'seed_memory' : 7563

    }

# Initialize random seed

np.random.seed(settings["seed_general"])
random.seed(settings["seed_general"])

# Initialize Atari simulator
simulator = AtariSimulator(settings)

# Initialize a deep net structure

class Convolution(Chain):

    def __init__(self):
        super(Convolution, self).__init__(
            l1=F.Convolution2D(settings['history_sizes'][0], 32, ksize=8, stride=4, nobias=False, wscale=np.sqrt(2)),
            l2=F.Convolution2D(32, 64, ksize=4, stride=2, nobias=False, wscale=np.sqrt(2)),
            l3=F.Convolution2D(64, 64, ksize=3, stride=1, nobias=False, wscale=np.sqrt(2)),
            l4=F.Linear(3136, 512, wscale = np.sqrt(2)),
            l5=F.Linear(512, simulator.n_actions, wscale = np.sqrt(2)),
        )

    def __call__(self, ohist, ahist):
        if len(ohist.data.shape) < 4:
            ohist = F.reshape(ohist,(1,4,84,84))
        h1 = F.relu(self.l1(ohist/255.0))
        h2 = F.relu(self.l2(h1))
        h3 = F.relu(self.l3(h2))
        h4 = F.relu(self.l4(h3))
        output = self.l5(h4)
        return output

net = Convolution()

# Initialize Learner with a Chainer backend

backend = ChainerBackend(settings)
backend.set_net(net)

learner = DQNLearner(settings, backend)

# Initialize memory

memory = ReplayMemoryHDF5(settings)

# Initialize Agent Framework

agent = DQNAgent(learner, memory, simulator, settings)

# Start training

agent.train()

# load net

# agent.learner.load_net(settings['save_dir']+'/net_750000.p')

# '''Plot training history'''

# plt.plot(range(len(learner.val_rewards)), learner.val_rewards)
# plt.xlabel("Evaluation episode (1000 transitions long, every 5000 training iter.)")
# plt.ylabel("Accumulated reward per episode")
# plt.xlim(0, 100)
# plt.ylim(-200, 200)
# plt.grid(True)
# plt.savefig(settings['save_dir'] + '_' + "evaluation_reward.svg", bbox_inches='tight')
# plt.close()

# plt.plot(range(len(learner.val_qval_avgs)), learner.val_qval_avgs)
# plt.xlabel("Evaluation episode  (1000 transitions long, every 5000 training iter.)")
# plt.ylabel("Average Q-value")
# plt.xlim(0, 100)
# plt.ylim(0, 20)
# plt.grid(True)
# plt.savefig(settings['save_dir'] + '_' + "evaluation_q_value.svg", bbox_inches='tight')
# plt.close()

# '''Plot value surface for beliefs'''
# ind_max = learner.val_rewards[0::5].index(max(learner.val_rewards[0::5]))
# ind_net = settings['save_every'] + ind_max * settings['save_every']
# learner.load_net(settings['save_dir']+'/net_%d.p' % int(ind_net))

# print(ind_net)
# print(ind_max*5)
# print(learner.val_rewards)
# print(learner.val_rewards[ind_max*5])

# '''Evaluate learned policy against the optimal heuristic policy'''
# np.random.seed(settings["seed_general"])

# print('Evaluating DQN agent...')
# print('(reward, MSE loss, mean Q-value, episodes - NA, time)')
# reward, MSE_loss, mean_Q_value, episodes, time, paths, actions, rewards = agent.evaluate(learner, simulator, 50000)
# print(reward, MSE_loss, mean_Q_value, episodes, time)
<|endoftext|>


<commit_before>from cereal import car
from common.conversions import Conversions as CV
from common.numpy_fast import mean
from opendbc.can.can_define import CANDefine
from opendbc.can.parser import CANParser
from selfdrive.car.interfaces import CarStateBase
from selfdrive.car.gm.values import DBC, AccState, CanBus, STEER_THRESHOLD

TransmissionType = car.CarParams.TransmissionType
NetworkLocation = car.CarParams.NetworkLocation

class CarState(CarStateBase):
  def __init__(self, CP):
    super().__init__(CP)
    can_define = CANDefine(DBC[CP.carFingerprint]["pt"])
    self.shifter_values = can_define.dv["ECMPRDNL2"]["PRNDL2"]
    self.loopback_lka_steering_cmd_updated = False
    self.camera_lka_steering_cmd_counter = 0
    self.buttons_counter = 0

  def update(self, pt_cp, cam_cp, loopback_cp):
    ret = car.CarState.new_message()

    self.prev_cruise_buttons = self.cruise_buttons
    self.cruise_buttons = pt_cp.vl["ASCMSteeringButton"]["ACCButtons"]
    self.buttons_counter = pt_cp.vl["ASCMSteeringButton"]["RollingCounter"]

    # Variables used for avoiding LKAS faults
    self.loopback_lka_steering_cmd_updated = len(loopback_cp.vl_all["ASCMLKASteeringCmd"]) > 0
    if self.CP.networkLocation == NetworkLocation.fwdCamera:
      self.camera_lka_steering_cmd_counter = cam_cp.vl["ASCMLKASteeringCmd"]["RollingCounter"]

    ret.wheelSpeeds = self.get_wheel_speeds(
      pt_cp.vl["EBCMWheelSpdFront"]["FLWheelSpd"],
      pt_cp.vl["EBCMWheelSpdFront"]["FRWheelSpd"],
      pt_cp.vl["EBCMWheelSpdRear"]["RLWheelSpd"],
      pt_cp.vl["EBCMWheelSpdRear"]["RRWheelSpd"],
    )
    ret.vEgoRaw = mean([ret.wheelSpeeds.fl, ret.wheelSpeeds.fr, ret.wheelSpeeds.rl, ret.wheelSpeeds.rr])
    ret.vEgo, ret.aEgo = self.update_speed_kf(ret.vEgoRaw)
    ret.standstill = ret.vEgoRaw < 0.01

    if pt_cp.vl["ECMPRDNL2"]["ManualMode"] == 1:
      ret.gearShifter = self.parse_gear_shifter("T")
    else:
      ret.gearShifter = self.parse_gear_shifter(self.shifter_values.get(pt_cp.vl["ECMPRDNL2"]["PRNDL2"], None))

    # Some Volt 2016-17 have loose brake pedal push rod retainers which causes the ECM to believe
    # that the brake is being intermittently pressed without user interaction.
    # To avoid a cruise fault we need to match the ECM's brake pressed signal and threshold
    # https://static.nhtsa.gov/odi/tsbs/2017/MC-10137629-9999.pdf
    ret.brake = pt_cp.vl["ECMAcceleratorPos"]["BrakePedalPos"] / 0xd0
    ret.brakePressed = pt_cp.vl["ECMAcceleratorPos"]["BrakePedalPos"] >= 8

    # Regen braking is braking
    if self.CP.transmissionType == TransmissionType.direct:
      ret.brakePressed = ret.brakePressed or pt_cp.vl["EBCMRegenPaddle"]["RegenPaddle"] != 0

    ret.gas = pt_cp.vl["AcceleratorPedal2"]["AcceleratorPedal2"] / 254.
    ret.gasPressed = ret.gas > 1e-5

    ret.steeringAngleDeg = pt_cp.vl["PSCMSteeringAngle"]["SteeringWheelAngle"]
    ret.steeringRateDeg = pt_cp.vl["PSCMSteeringAngle"]["SteeringWheelRate"]
    ret.steeringTorque = pt_cp.vl["PSCMStatus"]["LKADriverAppldTrq"]
    ret.steeringTorqueEps = pt_cp.vl["PSCMStatus"]["LKATorqueDelivered"]
    ret.steeringPressed = abs(ret.steeringTorque) > STEER_THRESHOLD

    # 0 inactive, 1 active, 2 temporarily limited, 3 failed
    self.lkas_status = pt_cp.vl["PSCMStatus"]["LKATorqueDeliveredStatus"]
    ret.steerFaultTemporary = self.lkas_status == 2
    ret.steerFaultPermanent = self.lkas_status == 3

    # 1 - open, 0 - closed
    ret.doorOpen = (pt_cp.vl["BCMDoorBeltStatus"]["FrontLeftDoor"] == 1 or
                    pt_cp.vl["BCMDoorBeltStatus"]["FrontRightDoor"] == 1 or
                    pt_cp.vl["BCMDoorBeltStatus"]["RearLeftDoor"] == 1 or
                    pt_cp.vl["BCMDoorBeltStatus"]["RearRightDoor"] == 1)

    # 1 - latched
    ret.seatbeltUnlatched = pt_cp.vl["BCMDoorBeltStatus"]["LeftSeatBelt"] == 0
    ret.leftBlinker = pt_cp.vl["BCMTurnSignals"]["TurnSignals"] == 1
    ret.rightBlinker = pt_cp.vl["BCMTurnSignals"]["TurnSignals"] == 2

    ret.parkingBrake = pt_cp.vl["VehicleIgnitionAlt"]["ParkBrake"] == 1
    ret.cruiseState.available = pt_cp.vl["ECMEngineStatus"]["CruiseMainOn"] != 0
    ret.espDisabled = pt_cp.vl["ESPStatus"]["TractionControlOn"] != 1
    ret.accFaulted = pt_cp.vl["AcceleratorPedal2"]["CruiseState"] == AccState.FAULTED

    ret.cruiseState.enabled = pt_cp.vl["AcceleratorPedal2"]["CruiseState"] != AccState.OFF
    ret.cruiseState.standstill = pt_cp.vl["AcceleratorPedal2"]["CruiseState"] == AccState.STANDSTILL
    if self.CP.networkLocation == NetworkLocation.fwdCamera:
      ret.cruiseState.speed = cam_cp.vl["ASCMActiveCruiseControlStatus"]["ACCSpeedSetpoint"] * CV.KPH_TO_MS

    return ret

  @staticmethod
  def get_cam_can_parser(CP):
    signals = []
    checks = []
    if CP.networkLocation == NetworkLocation.fwdCamera:
      signals += [
        ("RollingCounter", "ASCMLKASteeringCmd"),
        ("ACCSpeedSetpoint", "ASCMActiveCruiseControlStatus"),
      ]
      checks += [
        ("ASCMLKASteeringCmd", 10),
        ("ASCMActiveCruiseControlStatus", 25),
      ]

    return CANParser(DBC[CP.carFingerprint]["pt"], signals, checks, CanBus.CAMERA)

  @staticmethod
  def get_can_parser(CP):
    signals = [
      # sig_name, sig_address
      ("BrakePedalPos", "ECMAcceleratorPos"),
      ("FrontLeftDoor", "BCMDoorBeltStatus"),
      ("FrontRightDoor", "BCMDoorBeltStatus"),
      ("RearLeftDoor", "BCMDoorBeltStatus"),
      ("RearRightDoor", "BCMDoorBeltStatus"),
      ("LeftSeatBelt", "BCMDoorBeltStatus"),
      ("RightSeatBelt", "BCMDoorBeltStatus"),
      ("TurnSignals", "BCMTurnSignals"),
      ("AcceleratorPedal2", "AcceleratorPedal2"),
      ("CruiseState", "AcceleratorPedal2"),
      ("ACCButtons", "ASCMSteeringButton"),
      ("RollingCounter", "ASCMSteeringButton"),
      ("SteeringWheelAngle", "PSCMSteeringAngle"),
      ("SteeringWheelRate", "PSCMSteeringAngle"),
      ("FLWheelSpd", "EBCMWheelSpdFront"),
      ("FRWheelSpd", "EBCMWheelSpdFront"),
      ("RLWheelSpd", "EBCMWheelSpdRear"),
      ("RRWheelSpd", "EBCMWheelSpdRear"),
      ("PRNDL2", "ECMPRDNL2"),
      ("ManualMode", "ECMPRDNL2"),
      ("LKADriverAppldTrq", "PSCMStatus"),
      ("LKATorqueDelivered", "PSCMStatus"),
      ("LKATorqueDeliveredStatus", "PSCMStatus"),
      ("TractionControlOn", "ESPStatus"),
      ("ParkBrake", "VehicleIgnitionAlt"),
      ("CruiseMainOn", "ECMEngineStatus"),
    ]

    checks = [
      ("BCMTurnSignals", 1),
      ("ECMPRDNL2", 10),
      ("PSCMStatus", 10),
      ("ESPStatus", 10),
      ("BCMDoorBeltStatus", 10),
      ("VehicleIgnitionAlt", 10),
      ("EBCMWheelSpdFront", 20),
      ("EBCMWheelSpdRear", 20),
      ("AcceleratorPedal2", 33),
      ("ASCMSteeringButton", 33),
      ("ECMEngineStatus", 100),
      ("PSCMSteeringAngle", 100),
      ("ECMAcceleratorPos", 80),
    ]

    if CP.transmissionType == TransmissionType.direct:
      signals.append(("RegenPaddle", "EBCMRegenPaddle"))
      checks.append(("EBCMRegenPaddle", 50))

    return CANParser(DBC[CP.carFingerprint]["pt"], signals, checks, CanBus.POWERTRAIN)

  @staticmethod
  def get_loopback_can_parser(CP):
    signals = [
      ("RollingCounter", "ASCMLKASteeringCmd"),
    ]

    checks = [
      ("ASCMLKASteeringCmd", 10), # 10 Hz is the stock inactive rate (every 100ms).
      #                             While active 50 Hz (every 20 ms) is normal
      #                             EPS will tolerate around 200ms when active before faulting
    ]

    return CANParser(DBC[CP.carFingerprint]["pt"], signals, checks, CanBus.LOOPBACK)
<commit_msg>GM camera ACC: log stock aeb/fcw (#26017)<commit_after>from cereal import car
from common.conversions import Conversions as CV
from common.numpy_fast import mean
from opendbc.can.can_define import CANDefine
from opendbc.can.parser import CANParser
from selfdrive.car.interfaces import CarStateBase
from selfdrive.car.gm.values import DBC, AccState, CanBus, STEER_THRESHOLD

TransmissionType = car.CarParams.TransmissionType
NetworkLocation = car.CarParams.NetworkLocation

class CarState(CarStateBase):
  def __init__(self, CP):
    super().__init__(CP)
    can_define = CANDefine(DBC[CP.carFingerprint]["pt"])
    self.shifter_values = can_define.dv["ECMPRDNL2"]["PRNDL2"]
    self.loopback_lka_steering_cmd_updated = False
    self.camera_lka_steering_cmd_counter = 0
    self.buttons_counter = 0

  def update(self, pt_cp, cam_cp, loopback_cp):
    ret = car.CarState.new_message()

    self.prev_cruise_buttons = self.cruise_buttons
    self.cruise_buttons = pt_cp.vl["ASCMSteeringButton"]["ACCButtons"]
    self.buttons_counter = pt_cp.vl["ASCMSteeringButton"]["RollingCounter"]

    # Variables used for avoiding LKAS faults
    self.loopback_lka_steering_cmd_updated = len(loopback_cp.vl_all["ASCMLKASteeringCmd"]) > 0
    if self.CP.networkLocation == NetworkLocation.fwdCamera:
      self.camera_lka_steering_cmd_counter = cam_cp.vl["ASCMLKASteeringCmd"]["RollingCounter"]

    ret.wheelSpeeds = self.get_wheel_speeds(
      pt_cp.vl["EBCMWheelSpdFront"]["FLWheelSpd"],
      pt_cp.vl["EBCMWheelSpdFront"]["FRWheelSpd"],
      pt_cp.vl["EBCMWheelSpdRear"]["RLWheelSpd"],
      pt_cp.vl["EBCMWheelSpdRear"]["RRWheelSpd"],
    )
    ret.vEgoRaw = mean([ret.wheelSpeeds.fl, ret.wheelSpeeds.fr, ret.wheelSpeeds.rl, ret.wheelSpeeds.rr])
    ret.vEgo, ret.aEgo = self.update_speed_kf(ret.vEgoRaw)
    ret.standstill = ret.vEgoRaw < 0.01

    if pt_cp.vl["ECMPRDNL2"]["ManualMode"] == 1:
      ret.gearShifter = self.parse_gear_shifter("T")
    else:
      ret.gearShifter = self.parse_gear_shifter(self.shifter_values.get(pt_cp.vl["ECMPRDNL2"]["PRNDL2"], None))

    # Some Volt 2016-17 have loose brake pedal push rod retainers which causes the ECM to believe
    # that the brake is being intermittently pressed without user interaction.
    # To avoid a cruise fault we need to match the ECM's brake pressed signal and threshold
    # https://static.nhtsa.gov/odi/tsbs/2017/MC-10137629-9999.pdf
    ret.brake = pt_cp.vl["ECMAcceleratorPos"]["BrakePedalPos"] / 0xd0
    ret.brakePressed = pt_cp.vl["ECMAcceleratorPos"]["BrakePedalPos"] >= 8

    # Regen braking is braking
    if self.CP.transmissionType == TransmissionType.direct:
      ret.brakePressed = ret.brakePressed or pt_cp.vl["EBCMRegenPaddle"]["RegenPaddle"] != 0

    ret.gas = pt_cp.vl["AcceleratorPedal2"]["AcceleratorPedal2"] / 254.
    ret.gasPressed = ret.gas > 1e-5

    ret.steeringAngleDeg = pt_cp.vl["PSCMSteeringAngle"]["SteeringWheelAngle"]
    ret.steeringRateDeg = pt_cp.vl["PSCMSteeringAngle"]["SteeringWheelRate"]
    ret.steeringTorque = pt_cp.vl["PSCMStatus"]["LKADriverAppldTrq"]
    ret.steeringTorqueEps = pt_cp.vl["PSCMStatus"]["LKATorqueDelivered"]
    ret.steeringPressed = abs(ret.steeringTorque) > STEER_THRESHOLD

    # 0 inactive, 1 active, 2 temporarily limited, 3 failed
    self.lkas_status = pt_cp.vl["PSCMStatus"]["LKATorqueDeliveredStatus"]
    ret.steerFaultTemporary = self.lkas_status == 2
    ret.steerFaultPermanent = self.lkas_status == 3

    # 1 - open, 0 - closed
    ret.doorOpen = (pt_cp.vl["BCMDoorBeltStatus"]["FrontLeftDoor"] == 1 or
                    pt_cp.vl["BCMDoorBeltStatus"]["FrontRightDoor"] == 1 or
                    pt_cp.vl["BCMDoorBeltStatus"]["RearLeftDoor"] == 1 or
                    pt_cp.vl["BCMDoorBeltStatus"]["RearRightDoor"] == 1)

    # 1 - latched
    ret.seatbeltUnlatched = pt_cp.vl["BCMDoorBeltStatus"]["LeftSeatBelt"] == 0
    ret.leftBlinker = pt_cp.vl["BCMTurnSignals"]["TurnSignals"] == 1
    ret.rightBlinker = pt_cp.vl["BCMTurnSignals"]["TurnSignals"] == 2

    ret.parkingBrake = pt_cp.vl["VehicleIgnitionAlt"]["ParkBrake"] == 1
    ret.cruiseState.available = pt_cp.vl["ECMEngineStatus"]["CruiseMainOn"] != 0
    ret.espDisabled = pt_cp.vl["ESPStatus"]["TractionControlOn"] != 1
    ret.accFaulted = pt_cp.vl["AcceleratorPedal2"]["CruiseState"] == AccState.FAULTED

    ret.cruiseState.enabled = pt_cp.vl["AcceleratorPedal2"]["CruiseState"] != AccState.OFF
    ret.cruiseState.standstill = pt_cp.vl["AcceleratorPedal2"]["CruiseState"] == AccState.STANDSTILL
    if self.CP.networkLocation == NetworkLocation.fwdCamera:
      ret.cruiseState.speed = cam_cp.vl["ASCMActiveCruiseControlStatus"]["ACCSpeedSetpoint"] * CV.KPH_TO_MS

      ret.stockAeb = cam_cp.vl["AEBCmd"]["AEBCmdActive"] != 0
      ret.stockFcw = cam_cp.vl["ASCMActiveCruiseControlStatus"]["FCWAlert"] != 0

    return ret

  @staticmethod
  def get_cam_can_parser(CP):
    signals = []
    checks = []
    if CP.networkLocation == NetworkLocation.fwdCamera:
      signals += [
        ("AEBCmdActive", "AEBCmd"),
        ("RollingCounter", "ASCMLKASteeringCmd"),
        ("FCWAlert", "ASCMActiveCruiseControlStatus"),
        ("ACCSpeedSetpoint", "ASCMActiveCruiseControlStatus"),
      ]
      checks += [
        ("AEBCmd", 10),
        ("ASCMLKASteeringCmd", 10),
        ("ASCMActiveCruiseControlStatus", 25),
      ]

    return CANParser(DBC[CP.carFingerprint]["pt"], signals, checks, CanBus.CAMERA)

  @staticmethod
  def get_can_parser(CP):
    signals = [
      # sig_name, sig_address
      ("BrakePedalPos", "ECMAcceleratorPos"),
      ("FrontLeftDoor", "BCMDoorBeltStatus"),
      ("FrontRightDoor", "BCMDoorBeltStatus"),
      ("RearLeftDoor", "BCMDoorBeltStatus"),
      ("RearRightDoor", "BCMDoorBeltStatus"),
      ("LeftSeatBelt", "BCMDoorBeltStatus"),
      ("RightSeatBelt", "BCMDoorBeltStatus"),
      ("TurnSignals", "BCMTurnSignals"),
      ("AcceleratorPedal2", "AcceleratorPedal2"),
      ("CruiseState", "AcceleratorPedal2"),
      ("ACCButtons", "ASCMSteeringButton"),
      ("RollingCounter", "ASCMSteeringButton"),
      ("SteeringWheelAngle", "PSCMSteeringAngle"),
      ("SteeringWheelRate", "PSCMSteeringAngle"),
      ("FLWheelSpd", "EBCMWheelSpdFront"),
      ("FRWheelSpd", "EBCMWheelSpdFront"),
      ("RLWheelSpd", "EBCMWheelSpdRear"),
      ("RRWheelSpd", "EBCMWheelSpdRear"),
      ("PRNDL2", "ECMPRDNL2"),
      ("ManualMode", "ECMPRDNL2"),
      ("LKADriverAppldTrq", "PSCMStatus"),
      ("LKATorqueDelivered", "PSCMStatus"),
      ("LKATorqueDeliveredStatus", "PSCMStatus"),
      ("TractionControlOn", "ESPStatus"),
      ("ParkBrake", "VehicleIgnitionAlt"),
      ("CruiseMainOn", "ECMEngineStatus"),
    ]

    checks = [
      ("BCMTurnSignals", 1),
      ("ECMPRDNL2", 10),
      ("PSCMStatus", 10),
      ("ESPStatus", 10),
      ("BCMDoorBeltStatus", 10),
      ("VehicleIgnitionAlt", 10),
      ("EBCMWheelSpdFront", 20),
      ("EBCMWheelSpdRear", 20),
      ("AcceleratorPedal2", 33),
      ("ASCMSteeringButton", 33),
      ("ECMEngineStatus", 100),
      ("PSCMSteeringAngle", 100),
      ("ECMAcceleratorPos", 80),
    ]

    if CP.transmissionType == TransmissionType.direct:
      signals.append(("RegenPaddle", "EBCMRegenPaddle"))
      checks.append(("EBCMRegenPaddle", 50))

    return CANParser(DBC[CP.carFingerprint]["pt"], signals, checks, CanBus.POWERTRAIN)

  @staticmethod
  def get_loopback_can_parser(CP):
    signals = [
      ("RollingCounter", "ASCMLKASteeringCmd"),
    ]

    checks = [
      ("ASCMLKASteeringCmd", 10), # 10 Hz is the stock inactive rate (every 100ms).
      #                             While active 50 Hz (every 20 ms) is normal
      #                             EPS will tolerate around 200ms when active before faulting
    ]

    return CANParser(DBC[CP.carFingerprint]["pt"], signals, checks, CanBus.LOOPBACK)
<|endoftext|>


<commit_before>import cherrypy, re, os.path, cPickle as pickle, time, calendar

from base_proxy import BaseProxy
import tools.hashlib_shortcuts, tools.file

class Proxy(BaseProxy):
    default_remote_host = cherrypy.config['remote.host']

    reverse_host_map = {
        cherrypy.config['remote.host']: '',
    }

    def __init__(self):
        BaseProxy.__init__(self)

        self.html_comment_re = re.compile(r'<!--.*?-->', re.S)
        self.html_script_re = re.compile(r'<script.*?</script>', re.S)

    def perform(self, **kwargs):
        BaseProxy.perform(self, **kwargs)

        self._massage_remote_response()

    def _massage_remote_response(self):
        r = self._remote_response

        # kill no-cache and no-store directives, but note whether response was private or not.
        # private responses are not saved to disk later
        r.private = False
        if r.headers.has_key('pragma'):
            value = r.headers['pragma']
            if value == 'no-cache':
                del r.headers['pragma']
                r.private = True
        if r.headers.has_key('cache-control'):
            value = r.headers['cache-control']
            parts = [part.strip() for part in value.split(',')]
            new_parts = [part for part in parts if part not in ['no-cache', 'no-store']]
            if len(parts) != len(new_parts):
                if 'private' not in new_parts:
                    new_parts.append('private')
                new_value = ', '.join(new_parts)
                r.headers['cache-control'] = new_value
                r.private = True
        # quick hack: kill old expiration dates from private responses, without parsing dates
        if r.private and r.headers.has_key('expires'):
            del r.headers['expires']

        if r.headers['content-type'].lower().startswith('text/html'):
            content = r.content
            content = self.html_comment_re.sub('', content)
            search = cherrypy.config['remote.host']
            replace = cherrypy.request.headers.get('host') or self.__class__.default_remote_host
            content = content.replace(search, replace)
            #content = self.html_script_re.sub(lambda match: match.group(0).replace(search, replace), content)
            r.content = content

        return
        try:
            del r.headers['cache-control']
        except:
            pass
        try:
            del r.headers['expires']
        except:
            pass

class ContentWrapper:
    def __init__(self, content):
        self.content = content

class CachingProxy(Proxy):
    def perform_and_propagate(self, **kwargs):
        self._setup_cache_variables()
        content = self._find_in_cache()
        if content is None:
            response = Proxy.perform_and_propagate(self, **kwargs)
            if not response.private:
                self._save_to_cache(response)
        else:
            response = ContentWrapper(content)
        return response

    def _setup_cache_variables(self):
        self.cache_absolute_path = self.cache_absolute_path_meta = None
        r = cherrypy.request
        if r.query_string:
            hashed_qs = tools.hashlib_shortcuts.md5_hexdigest(r.query_string)
            relative_path = r.path_info + '::' + hashed_qs
        else:
            relative_path = r.path_info
        if relative_path.find('..') >= 0:
            raise ValueError('Suspicious request relative path: %s' % relative_path)
        assert relative_path[0] == '/'
        relative_path = relative_path[1:]
        if relative_path:
            assert relative_path[0] != '/'
            self.cache_absolute_path = os.path.join(cherrypy.config['local.cache.dir'], relative_path)
            self.cache_absolute_path_meta = self.cache_absolute_path + '.meta'

    def _find_in_cache(self):
        if self.cache_absolute_path_meta is not None and os.path.exists(self.cache_absolute_path_meta):
            headers = pickle.loads(tools.file.read(self.cache_absolute_path_meta))
            expires = headers['x-expires-timestamp']
            now = time.time()
            if expires >= now:
                for key, value in headers.items():
                    cherrypy.response.headers[key] = value
                content = tools.file.read(self.cache_absolute_path)
                return content

    def _save_to_cache(self, response):
        if self.cache_absolute_path is None:
            return

        headers = response.headers
        expires_at = None
        if headers.has_key('cache-control'):
            cache_control = headers['cache-control']
            parts = [part.strip() for part in cache_control.split(',')]
            for part in parts:
                if part.startswith('max-age='):
                    age = int(part[8:])
                    expires_at = int(time.time()) + age
                    break
        if expires_at is None and headers.has_key('expires'):
            expires_at = calendar.timegm(time.strptime(headers['expires'], '%a, %d %b %Y %H:%M:%S %Z'))
        if expires_at is not None:
            headers['x-expires-timestamp'] = expires_at
            dir = os.path.dirname(self.cache_absolute_path)
            if not os.path.exists(dir):
                tools.file.safe_mkdirs(dir)
            tools.file.safe_write(self.cache_absolute_path, response.content)
            tools.file.safe_write(self.cache_absolute_path_meta, pickle.dumps(headers))
<commit_msg>jira-proxy: removed debug info<commit_after>import cherrypy, re, os.path, cPickle as pickle, time, calendar

from base_proxy import BaseProxy
import tools.hashlib_shortcuts, tools.file

class Proxy(BaseProxy):
    default_remote_host = cherrypy.config['remote.host']

    reverse_host_map = {
        cherrypy.config['remote.host']: '',
    }

    def __init__(self):
        BaseProxy.__init__(self)

        self.html_comment_re = re.compile(r'<!--.*?-->', re.S)
        self.html_script_re = re.compile(r'<script.*?</script>', re.S)

    def perform(self, **kwargs):
        BaseProxy.perform(self, **kwargs)

        self._massage_remote_response()

    def _massage_remote_response(self):
        r = self._remote_response

        # kill no-cache and no-store directives, but note whether response was private or not.
        # private responses are not saved to disk later
        r.private = False
        if r.headers.has_key('pragma'):
            value = r.headers['pragma']
            if value == 'no-cache':
                del r.headers['pragma']
                r.private = True
        if r.headers.has_key('cache-control'):
            value = r.headers['cache-control']
            parts = [part.strip() for part in value.split(',')]
            new_parts = [part for part in parts if part not in ['no-cache', 'no-store']]
            if len(parts) != len(new_parts):
                if 'private' not in new_parts:
                    new_parts.append('private')
                new_value = ', '.join(new_parts)
                r.headers['cache-control'] = new_value
                r.private = True
        # quick hack: kill old expiration dates from private responses, without parsing dates
        if r.private and r.headers.has_key('expires'):
            del r.headers['expires']

        if r.headers['content-type'].lower().startswith('text/html'):
            content = r.content
            content = self.html_comment_re.sub('', content)
            search = cherrypy.config['remote.host']
            replace = cherrypy.request.headers.get('host') or self.__class__.default_remote_host
            content = content.replace(search, replace)
            #content = self.html_script_re.sub(lambda match: match.group(0).replace(search, replace), content)
            r.content = content

class ContentWrapper:
    def __init__(self, content):
        self.content = content

class CachingProxy(Proxy):
    def perform_and_propagate(self, **kwargs):
        self._setup_cache_variables()
        content = self._find_in_cache()
        if content is None:
            response = Proxy.perform_and_propagate(self, **kwargs)
            if not response.private:
                self._save_to_cache(response)
        else:
            response = ContentWrapper(content)
        return response

    def _setup_cache_variables(self):
        self.cache_absolute_path = self.cache_absolute_path_meta = None
        r = cherrypy.request
        if r.query_string:
            hashed_qs = tools.hashlib_shortcuts.md5_hexdigest(r.query_string)
            relative_path = r.path_info + '::' + hashed_qs
        else:
            relative_path = r.path_info
        if relative_path.find('..') >= 0:
            raise ValueError('Suspicious request relative path: %s' % relative_path)
        assert relative_path[0] == '/'
        relative_path = relative_path[1:]
        if relative_path:
            assert relative_path[0] != '/'
            self.cache_absolute_path = os.path.join(cherrypy.config['local.cache.dir'], relative_path)
            self.cache_absolute_path_meta = self.cache_absolute_path + '.meta'

    def _find_in_cache(self):
        if self.cache_absolute_path_meta is not None and os.path.exists(self.cache_absolute_path_meta):
            headers = pickle.loads(tools.file.read(self.cache_absolute_path_meta))
            expires = headers['x-expires-timestamp']
            now = time.time()
            if expires >= now:
                for key, value in headers.items():
                    cherrypy.response.headers[key] = value
                content = tools.file.read(self.cache_absolute_path)
                return content

    def _save_to_cache(self, response):
        if self.cache_absolute_path is None:
            return

        headers = response.headers
        expires_at = None
        if headers.has_key('cache-control'):
            cache_control = headers['cache-control']
            parts = [part.strip() for part in cache_control.split(',')]
            for part in parts:
                if part.startswith('max-age='):
                    age = int(part[8:])
                    expires_at = int(time.time()) + age
                    break
        if expires_at is None and headers.has_key('expires'):
            expires_at = calendar.timegm(time.strptime(headers['expires'], '%a, %d %b %Y %H:%M:%S %Z'))
        if expires_at is not None:
            headers['x-expires-timestamp'] = expires_at
            dir = os.path.dirname(self.cache_absolute_path)
            if not os.path.exists(dir):
                tools.file.safe_mkdirs(dir)
            tools.file.safe_write(self.cache_absolute_path, response.content)
            tools.file.safe_write(self.cache_absolute_path_meta, pickle.dumps(headers))
<|endoftext|>


<commit_before># Copyright 2022 Planet Labs PBC.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""The Planet Data CLI."""
from typing import List, Optional
from contextlib import asynccontextmanager

import click

from planet import data_filter, DataClient
from planet.clients.data import (SEARCH_SORT,
                                 SEARCH_SORT_DEFAULT,
                                 STATS_INTERVAL)
from planet.specs import get_item_types, SpecificationException

from . import types
from .cmds import coro, translate_exceptions
from .io import echo_json
from .options import limit, pretty
from .session import CliSession

ALL_ITEM_TYPES = get_item_types()
valid_item_string = "Valid entries for ITEM_TYPES: " + "|".join(ALL_ITEM_TYPES)

@asynccontextmanager
async def data_client(ctx):
    auth = ctx.obj['AUTH']
    base_url = ctx.obj['BASE_URL']
    async with CliSession(auth=auth) as sess:
        cl = DataClient(sess, base_url=base_url)
        yield cl

@click.group()
@click.pass_context
@click.option('-u',
              '--base-url',
              default=None,
              help='Assign custom base Orders API URL.')
def data(ctx, base_url):
    '''Commands for interacting with the Data API'''
    ctx.obj['AUTH'] = None
    ctx.obj['BASE_URL'] = base_url

# TODO: filter().
def geom_to_filter(ctx, param, value: Optional[dict]) -> Optional[dict]:
    return data_filter.geometry_filter(value) if value else None

def assets_to_filter(ctx, param, assets: List[str]) -> Optional[dict]:
    # TODO: validate and normalize
    return data_filter.asset_filter(assets) if assets else None

def check_item_types(ctx, param, item_types) -> Optional[List[dict]]:
    # Set difference between given item types and all item types
    invalid_item_types = set([item.lower() for item in item_types]) - set(
        [a.lower() for a in ALL_ITEM_TYPES])
    if invalid_item_types:
        raise SpecificationException(invalid_item_types,
                                     ALL_ITEM_TYPES,
                                     'item_type')
    else:
        return item_types

def date_range_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, comp, value = obj
        kwargs = {'field_name': field, comp: value}
        return data_filter.date_range_filter(**kwargs)

    return [_func(v) for v in values] if values else None

def range_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, comp, value = obj
        kwargs = {'field_name': field, comp: value}
        return data_filter.range_filter(**kwargs)

    return [_func(v) for v in values] if values else None

def update_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, comp, value = obj
        kwargs = {'field_name': field, comp: value}
        return data_filter.update_filter(**kwargs)

    return [_func(v) for v in values] if values else None

def number_in_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, values = obj
        return data_filter.number_in_filter(field_name=field, values=values)

    return [_func(v) for v in values] if values else None

def string_in_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, values = obj
        return data_filter.string_in_filter(field_name=field, values=values)

    return [_func(v) for v in values] if values else None

@data.command()
@click.pass_context
@translate_exceptions
@click.option('--asset',
              type=types.CommaSeparatedString(),
              default=None,
              callback=assets_to_filter,
              help="""Filter to items with one or more of specified assets.
    TEXT is a comma-separated list of entries.
    When multiple entries are specified, an implicit 'or' logic is applied.""")
@click.option('--date-range',
              type=click.Tuple(
                  [types.Field(), types.Comparison(), types.DateTime()]),
              callback=date_range_to_filter,
              multiple=True,
              help="""Filter by date range in field.
    FIELD is the name of the field to filter on.
    COMP can be lt, lte, gt, or gte.
    DATETIME can be an RFC3339 or ISO 8601 string.""")
@click.option('--geom',
              type=types.JSON(),
              callback=geom_to_filter,
              help="""Filter to items that overlap a given geometry. Can be a
              json string, filename, or '-' for stdin.""")
@click.option('--number-in',
              type=click.Tuple([types.Field(), types.CommaSeparatedFloat()]),
              multiple=True,
              callback=number_in_to_filter,
              help="""Filter field by numeric in.
    FIELD is the name of the field to filter on.
    VALUE is a comma-separated list of entries.
    When multiple entries are specified, an implicit 'or' logic is applied.""")
@click.option('--permission',
              type=bool,
              default=True,
              show_default=True,
              help='Filter to assets with download permissions.')
@click.option('--range',
              'nrange',
              type=click.Tuple([types.Field(), types.Comparison(), float]),
              callback=range_to_filter,
              multiple=True,
              help="""Filter by number range in field.
    FIELD is the name of the field to filter on.
    COMP can be lt, lte, gt, or gte.""")
@click.option('--std-quality',
              type=bool,
              default=True,
              show_default=True,
              help='Filter to standard quality.')
@click.option('--string-in',
              type=click.Tuple([types.Field(), types.CommaSeparatedString()]),
              multiple=True,
              callback=string_in_to_filter,
              help="""Filter field by numeric in.
    FIELD is the name of the field to filter on.
    VALUE is a comma-separated list of entries.
    When multiple entries are specified, an implicit 'or' logic is applied.""")
@click.option(
    '--update',
    type=click.Tuple([types.Field(), types.GTComparison(), types.DateTime()]),
    callback=update_to_filter,
    multiple=True,
    help="""Filter to items with changes to a specified field value made after
    a specified date.
    FIELD is the name of the field to filter on.
    COMP can be gt or gte.
    DATETIME can be an RFC3339 or ISO 8601 string.""")
@pretty
def filter(ctx,
           asset,
           date_range,
           geom,
           number_in,
           nrange,
           string_in,
           update,
           permission,
           pretty,
           std_quality):
    """Create a structured item search filter.

    This command provides basic functionality for specifying a filter by
    creating an AndFilter with the filters identified with the options as
    inputs. This is only a subset of the complex filtering supported by the
    API. For advanced filter creation, either create the filter by hand or use
    the Python API.
    """
    permission = data_filter.permission_filter() if permission else None
    std_quality = data_filter.std_quality_filter() if std_quality else None

    filter_options = (asset,
                      date_range,
                      geom,
                      number_in,
                      nrange,
                      string_in,
                      update,
                      permission,
                      std_quality)

    # options allowing multiples are broken up so one filter is created for
    # each time the option is specified
    # unspecified options are skipped
    filters = []
    for f in filter_options:
        if f:
            if isinstance(f, list):
                filters.extend(f)
            else:
                filters.append(f)

    filt = data_filter.and_filter(filters)
    echo_json(filt, pretty)

@data.command(epilog=valid_item_string)
@click.pass_context
@translate_exceptions
@coro
@click.argument("item_types",
                type=types.CommaSeparatedString(),
                callback=check_item_types)
@click.argument("filter", type=types.JSON(), default="-", required=False)
@limit
@click.option('--name', type=str, help='Name of the saved search.')
@click.option('--sort',
              type=click.Choice(SEARCH_SORT),
              default=SEARCH_SORT_DEFAULT,
              show_default=True,
              help='Field and direction to order results by.')
@pretty
async def search(ctx, item_types, filter, limit, name, sort, pretty):
    """Execute a structured item search.

    This function outputs a series of GeoJSON descriptions, one for each of the
    returned items, optionally pretty-printed.

    ITEM_TYPES is a comma-separated list of item-types to search.

    FILTER must be JSON and can be specified a json string, filename, or '-'
    for stdin. It defaults to reading from stdin.

    Quick searches are stored for approximately 30 days and the --name
    parameter will be applied to the stored quick search.
    """
    async with data_client(ctx) as cl:
        items = await cl.search(item_types,
                                filter,
                                name=name,
                                sort=sort,
                                limit=limit)
        async for item in items:
            echo_json(item, pretty)

@data.command(epilog=valid_item_string)
@click.pass_context
@translate_exceptions
@coro
@click.argument('name')
@click.argument("item_types",
                type=types.CommaSeparatedString(),
                callback=check_item_types)
@click.argument("filter", type=types.JSON(), default="-", required=False)
@click.option('--daily-email',
              is_flag=True,
              help='Send a daily email when new results are added.')
@pretty
async def search_create(ctx, name, item_types, filter, daily_email, pretty):
    """Create a new saved structured item search.

    This function outputs a full JSON description of the created search,
    optionally pretty-printed.

    NAME is the name to give the search.

    ITEM_TYPES is a comma-separated list of item-types to search.

    FILTER must be JSON and can be specified a json string, filename, or '-'
    for stdin. It defaults to reading from stdin.
    """
    async with data_client(ctx) as cl:
        items = await cl.create_search(name=name,
                                       item_types=item_types,
                                       search_filter=filter,
                                       enable_email=daily_email)
        echo_json(items, pretty)

@data.command(epilog=valid_item_string)
@click.pass_context
@translate_exceptions
@coro
@click.argument("item_types",
                type=types.CommaSeparatedString(),
                callback=check_item_types)
@click.argument('interval', type=click.Choice(STATS_INTERVAL))
@click.argument("filter", type=types.JSON(), default="-", required=False)
async def stats(ctx, item_types, interval, filter):
    """Get a bucketed histogram of items matching the filter.

    This function returns a bucketed histogram of results based on the
    item_types, interval, and json filter specified (using file or stdin).

    """
    async with data_client(ctx) as cl:
        items = await cl.get_stats(item_types=item_types,
                                   interval=interval,
                                   search_filter=filter)
        echo_json(items)

@data.command()
@click.pass_context
@translate_exceptions
@coro
@pretty
@click.argument('search_id')
async def search_get(ctx, search_id, pretty):
    """Get a saved search.

    This function outputs a full JSON description of the identified saved
    search, optionally pretty-printed.
    """
    async with data_client(ctx) as cl:
        items = await cl.get_search(search_id)
        echo_json(items, pretty)

# TODO: search_update()".
# TODO: search_delete()".
# TODO: search_run()".
# TODO: item_get()".
# TODO: asset_activate()".
# TODO: asset_wait()".
# TODO: asset_download()".
<commit_msg>Validated item type with specs code.<commit_after># Copyright 2022 Planet Labs PBC.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""The Planet Data CLI."""
from typing import List, Optional
from contextlib import asynccontextmanager

import click

from planet import data_filter, DataClient
from planet.clients.data import (SEARCH_SORT,
                                 SEARCH_SORT_DEFAULT,
                                 STATS_INTERVAL)
from planet.specs import get_item_types, validate_item_type

from . import types
from .cmds import coro, translate_exceptions
from .io import echo_json
from .options import limit, pretty
from .session import CliSession

ALL_ITEM_TYPES = get_item_types()
valid_item_string = "Valid entries for ITEM_TYPES: " + "|".join(ALL_ITEM_TYPES)

@asynccontextmanager
async def data_client(ctx):
    auth = ctx.obj['AUTH']
    base_url = ctx.obj['BASE_URL']
    async with CliSession(auth=auth) as sess:
        cl = DataClient(sess, base_url=base_url)
        yield cl

@click.group()
@click.pass_context
@click.option('-u',
              '--base-url',
              default=None,
              help='Assign custom base Orders API URL.')
def data(ctx, base_url):
    '''Commands for interacting with the Data API'''
    ctx.obj['AUTH'] = None
    ctx.obj['BASE_URL'] = base_url

# TODO: filter().
def geom_to_filter(ctx, param, value: Optional[dict]) -> Optional[dict]:
    return data_filter.geometry_filter(value) if value else None

def assets_to_filter(ctx, param, assets: List[str]) -> Optional[dict]:
    # TODO: validate and normalize
    return data_filter.asset_filter(assets) if assets else None

def check_item_types(ctx, param, item_types) -> Optional[List[dict]]:
    for item_type in item_types:
        validate_item_type(item_type)
    return item_types

def date_range_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, comp, value = obj
        kwargs = {'field_name': field, comp: value}
        return data_filter.date_range_filter(**kwargs)

    return [_func(v) for v in values] if values else None

def range_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, comp, value = obj
        kwargs = {'field_name': field, comp: value}
        return data_filter.range_filter(**kwargs)

    return [_func(v) for v in values] if values else None

def update_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, comp, value = obj
        kwargs = {'field_name': field, comp: value}
        return data_filter.update_filter(**kwargs)

    return [_func(v) for v in values] if values else None

def number_in_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, values = obj
        return data_filter.number_in_filter(field_name=field, values=values)

    return [_func(v) for v in values] if values else None

def string_in_to_filter(ctx, param, values) -> Optional[List[dict]]:

    def _func(obj):
        field, values = obj
        return data_filter.string_in_filter(field_name=field, values=values)

    return [_func(v) for v in values] if values else None

@data.command()
@click.pass_context
@translate_exceptions
@click.option('--asset',
              type=types.CommaSeparatedString(),
              default=None,
              callback=assets_to_filter,
              help="""Filter to items with one or more of specified assets.
    TEXT is a comma-separated list of entries.
    When multiple entries are specified, an implicit 'or' logic is applied.""")
@click.option('--date-range',
              type=click.Tuple(
                  [types.Field(), types.Comparison(), types.DateTime()]),
              callback=date_range_to_filter,
              multiple=True,
              help="""Filter by date range in field.
    FIELD is the name of the field to filter on.
    COMP can be lt, lte, gt, or gte.
    DATETIME can be an RFC3339 or ISO 8601 string.""")
@click.option('--geom',
              type=types.JSON(),
              callback=geom_to_filter,
              help="""Filter to items that overlap a given geometry. Can be a
              json string, filename, or '-' for stdin.""")
@click.option('--number-in',
              type=click.Tuple([types.Field(), types.CommaSeparatedFloat()]),
              multiple=True,
              callback=number_in_to_filter,
              help="""Filter field by numeric in.
    FIELD is the name of the field to filter on.
    VALUE is a comma-separated list of entries.
    When multiple entries are specified, an implicit 'or' logic is applied.""")
@click.option('--permission',
              type=bool,
              default=True,
              show_default=True,
              help='Filter to assets with download permissions.')
@click.option('--range',
              'nrange',
              type=click.Tuple([types.Field(), types.Comparison(), float]),
              callback=range_to_filter,
              multiple=True,
              help="""Filter by number range in field.
    FIELD is the name of the field to filter on.
    COMP can be lt, lte, gt, or gte.""")
@click.option('--std-quality',
              type=bool,
              default=True,
              show_default=True,
              help='Filter to standard quality.')
@click.option('--string-in',
              type=click.Tuple([types.Field(), types.CommaSeparatedString()]),
              multiple=True,
              callback=string_in_to_filter,
              help="""Filter field by numeric in.
    FIELD is the name of the field to filter on.
    VALUE is a comma-separated list of entries.
    When multiple entries are specified, an implicit 'or' logic is applied.""")
@click.option(
    '--update',
    type=click.Tuple([types.Field(), types.GTComparison(), types.DateTime()]),
    callback=update_to_filter,
    multiple=True,
    help="""Filter to items with changes to a specified field value made after
    a specified date.
    FIELD is the name of the field to filter on.
    COMP can be gt or gte.
    DATETIME can be an RFC3339 or ISO 8601 string.""")
@pretty
def filter(ctx,
           asset,
           date_range,
           geom,
           number_in,
           nrange,
           string_in,
           update,
           permission,
           pretty,
           std_quality):
    """Create a structured item search filter.

    This command provides basic functionality for specifying a filter by
    creating an AndFilter with the filters identified with the options as
    inputs. This is only a subset of the complex filtering supported by the
    API. For advanced filter creation, either create the filter by hand or use
    the Python API.
    """
    permission = data_filter.permission_filter() if permission else None
    std_quality = data_filter.std_quality_filter() if std_quality else None

    filter_options = (asset,
                      date_range,
                      geom,
                      number_in,
                      nrange,
                      string_in,
                      update,
                      permission,
                      std_quality)

    # options allowing multiples are broken up so one filter is created for
    # each time the option is specified
    # unspecified options are skipped
    filters = []
    for f in filter_options:
        if f:
            if isinstance(f, list):
                filters.extend(f)
            else:
                filters.append(f)

    filt = data_filter.and_filter(filters)
    echo_json(filt, pretty)

@data.command(epilog=valid_item_string)
@click.pass_context
@translate_exceptions
@coro
@click.argument("item_types",
                type=types.CommaSeparatedString(),
                callback=check_item_types)
@click.argument("filter", type=types.JSON(), default="-", required=False)
@limit
@click.option('--name', type=str, help='Name of the saved search.')
@click.option('--sort',
              type=click.Choice(SEARCH_SORT),
              default=SEARCH_SORT_DEFAULT,
              show_default=True,
              help='Field and direction to order results by.')
@pretty
async def search(ctx, item_types, filter, limit, name, sort, pretty):
    """Execute a structured item search.

    This function outputs a series of GeoJSON descriptions, one for each of the
    returned items, optionally pretty-printed.

    ITEM_TYPES is a comma-separated list of item-types to search.

    FILTER must be JSON and can be specified a json string, filename, or '-'
    for stdin. It defaults to reading from stdin.

    Quick searches are stored for approximately 30 days and the --name
    parameter will be applied to the stored quick search.
    """
    async with data_client(ctx) as cl:
        items = await cl.search(item_types,
                                filter,
                                name=name,
                                sort=sort,
                                limit=limit)
        async for item in items:
            echo_json(item, pretty)

@data.command(epilog=valid_item_string)
@click.pass_context
@translate_exceptions
@coro
@click.argument('name')
@click.argument("item_types",
                type=types.CommaSeparatedString(),
                callback=check_item_types)
@click.argument("filter", type=types.JSON(), default="-", required=False)
@click.option('--daily-email',
              is_flag=True,
              help='Send a daily email when new results are added.')
@pretty
async def search_create(ctx, name, item_types, filter, daily_email, pretty):
    """Create a new saved structured item search.

    This function outputs a full JSON description of the created search,
    optionally pretty-printed.

    NAME is the name to give the search.

    ITEM_TYPES is a comma-separated list of item-types to search.

    FILTER must be JSON and can be specified a json string, filename, or '-'
    for stdin. It defaults to reading from stdin.
    """
    async with data_client(ctx) as cl:
        items = await cl.create_search(name=name,
                                       item_types=item_types,
                                       search_filter=filter,
                                       enable_email=daily_email)
        echo_json(items, pretty)

@data.command(epilog=valid_item_string)
@click.pass_context
@translate_exceptions
@coro
@click.argument("item_types",
                type=types.CommaSeparatedString(),
                callback=check_item_types)
@click.argument('interval', type=click.Choice(STATS_INTERVAL))
@click.argument("filter", type=types.JSON(), default="-", required=False)
async def stats(ctx, item_types, interval, filter):
    """Get a bucketed histogram of items matching the filter.

    This function returns a bucketed histogram of results based on the
    item_types, interval, and json filter specified (using file or stdin).

    """
    async with data_client(ctx) as cl:
        items = await cl.get_stats(item_types=item_types,
                                   interval=interval,
                                   search_filter=filter)
        echo_json(items)

@data.command()
@click.pass_context
@translate_exceptions
@coro
@pretty
@click.argument('search_id')
async def search_get(ctx, search_id, pretty):
    """Get a saved search.

    This function outputs a full JSON description of the identified saved
    search, optionally pretty-printed.
    """
    async with data_client(ctx) as cl:
        items = await cl.get_search(search_id)
        echo_json(items, pretty)

# TODO: search_update()".
# TODO: search_delete()".
# TODO: search_run()".
# TODO: item_get()".
# TODO: asset_activate()".
# TODO: asset_wait()".
# TODO: asset_download()".
<|endoftext|>


<commit_before>import json
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict
from django.conf.urls import patterns, url
from django.core.paginator import EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from avocado.core.paginator import BufferedPaginator
from avocado.formatters import RawFormatter
from avocado.export import HTMLExporter
from restlib2.params import Parametizer, param_cleaners
from .base import BaseResource

class PreviewParametizer(Parametizer):
    page = 1
    per_page = 50

    def clean_page(self, value):
        return param_cleaners.clean_int(value)

    def clean_per_page(self, value):
        return param_cleaners.clean_int(value)

class PreviewResource(BaseResource):
    """Resource for *previewing* data prior to exporting.

    Data is formatted using a JSON+HTML exporter which prefers HTML formatted
    or plain strings. Browser-based clients can consume the JSON and render
    the HTML for previewing.
    """
    parametizer = PreviewParametizer

    def get(self, request):
        uri = request.build_absolute_uri

        params = self.get_params(request)

        page = params.get('page')
        per_page = params.get('per_page')

        view = self.get_view(request)
        context = self.get_context(request)

        try:
            per_page = int(per_page)
        except (ValueError, TypeError):
            per_page = self.param_defaults.get('per_page')

        # Create the queryset based off the context. Note, this does not
        # take a `tree` param to customize the root model.
        queryset = context.apply().distinct()

        # For new contexts, `count` will be `None`
        if context.count is None:
            context.count = queryset.count()
            context.save()

        # Standard pagination components. A buffered paginator is used
        # here which takes a pre-computed count to same a bit of performance.
        # Otherwise the Paginator class itself would execute a count on
        # initialization.
        paginator = BufferedPaginator(context.count, per_page=per_page)

        try:
            page = paginator.page(page)
        except PageNotAnInteger:
            page = paginator.page(1)
        except EmptyPage:
            page = paginator.page(paginator.num_pages)

        # Get the current offset
        offset = page.offset()

        # Apply the view to the queryset. This was not applied before since
        # the distinct count may have been computed if this was a new context.
        # Include the primary key in this case since these objects a parsed
        # and dealt with programmatically downstream (due to the pagination)
        queryset = view.apply(queryset)

        # Slice and prepare as a raw query (note: this is a
        # ModelTreeQuerySet method and is not built-in to Django)
        iterator = queryset[offset:offset + per_page].raw()

        # Apply the view to the exporter class to get the formatted output
        exporter = HTMLExporter(view)

        # Insert formatter to process the primary key as a raw value
        # TODO: this is not terribly elegant
        pk_name = queryset.model._meta.pk.name
        exporter.params.insert(0, (RawFormatter(keys=[pk_name]), 1))
        exporter.row_length += 1

        # Build up the header keys.
        # TODO: This is flawed since it assumes the output columns
        # of exporter will be one-to-one with the concepts. This should
        # be built during the first iteration of the read, but would also
        # depend on data to exist!
        header = []
        view_node = view.parse()
        ordering = OrderedDict(view_node.ordering)

        for concept in view_node.columns:
            obj = {'id': concept.id, 'name': concept.name}
            if concept.id in ordering:
                obj['direction'] = ordering[concept.id]
            header.append(obj)

        objects = []
        for row in exporter.read(iterator):
            pk = None
            values = []
            for i, output in enumerate(row):
                if i == 0:
                    pk = output[pk_name]
                else:
                    values.extend(output.values())
            objects.append({'pk': pk, 'values': values})

        # Various other model options
        opts = queryset.model._meta
        model_name = opts.verbose_name.format()
        model_name_plural = opts.verbose_name_plural.format()

        data = {
            'keys': header,
            'objects': objects,
            'object_name': model_name,
            'object_name_plural': model_name_plural,
            'num_pages': paginator.num_pages,
            'page_num': page.number,
        }

        # Augment previous and next page links if other pages exist
        links = {
            'self': {
                'href': uri(reverse('serrano:data:preview') + '?page=' + \
                    str(page.number)),
            },
            'base': {
                'href': uri(reverse('serrano:data:preview')),
            }
        }

        if page.number != 1:
            links['prev'] = {
                'href': uri(reverse('serrano:data:preview') + '?page=' + \
                    str(page.number - 1)),
            }

        if page.number < paginator.num_pages - 1:
            links['next'] = {
                'href': uri(reverse('serrano:data:preview') + '?page=' + \
                    str(page.number + 1)),
            }

        data['_links'] = links

        return data

    post = get

preview_resource = PreviewResource()

# Resource endpoints
urlpatterns = patterns('',
    url(r'^$', preview_resource, name='preview'),
)
<commit_msg>Return per_page in preview metadata<commit_after>import json
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict
from django.conf.urls import patterns, url
from django.core.paginator import EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from avocado.core.paginator import BufferedPaginator
from avocado.formatters import RawFormatter
from avocado.export import HTMLExporter
from restlib2.params import Parametizer, param_cleaners
from .base import BaseResource

class PreviewParametizer(Parametizer):
    page = 1
    per_page = 50

    def clean_page(self, value):
        return param_cleaners.clean_int(value)

    def clean_per_page(self, value):
        return param_cleaners.clean_int(value)

class PreviewResource(BaseResource):
    """Resource for *previewing* data prior to exporting.

    Data is formatted using a JSON+HTML exporter which prefers HTML formatted
    or plain strings. Browser-based clients can consume the JSON and render
    the HTML for previewing.
    """
    parametizer = PreviewParametizer

    def get(self, request):
        uri = request.build_absolute_uri

        params = self.get_params(request)

        page = params.get('page')
        per_page = params.get('per_page')

        view = self.get_view(request)
        context = self.get_context(request)

        try:
            per_page = int(per_page)
        except (ValueError, TypeError):
            per_page = self.param_defaults.get('per_page')

        # Create the queryset based off the context. Note, this does not
        # take a `tree` param to customize the root model.
        queryset = context.apply().distinct()

        # For new contexts, `count` will be `None`
        if context.count is None:
            context.count = queryset.count()
            context.save()

        # Standard pagination components. A buffered paginator is used
        # here which takes a pre-computed count to same a bit of performance.
        # Otherwise the Paginator class itself would execute a count on
        # initialization.
        paginator = BufferedPaginator(context.count, per_page=per_page)

        try:
            page = paginator.page(page)
        except PageNotAnInteger:
            page = paginator.page(1)
        except EmptyPage:
            page = paginator.page(paginator.num_pages)

        # Get the current offset
        offset = page.offset()

        # Apply the view to the queryset. This was not applied before since
        # the distinct count may have been computed if this was a new context.
        # Include the primary key in this case since these objects a parsed
        # and dealt with programmatically downstream (due to the pagination)
        queryset = view.apply(queryset)

        # Slice and prepare as a raw query (note: this is a
        # ModelTreeQuerySet method and is not built-in to Django)
        iterator = queryset[offset:offset + per_page].raw()

        # Apply the view to the exporter class to get the formatted output
        exporter = HTMLExporter(view)

        # Insert formatter to process the primary key as a raw value
        # TODO: this is not terribly elegant
        pk_name = queryset.model._meta.pk.name
        exporter.params.insert(0, (RawFormatter(keys=[pk_name]), 1))
        exporter.row_length += 1

        # Build up the header keys.
        # TODO: This is flawed since it assumes the output columns
        # of exporter will be one-to-one with the concepts. This should
        # be built during the first iteration of the read, but would also
        # depend on data to exist!
        header = []
        view_node = view.parse()
        ordering = OrderedDict(view_node.ordering)

        for concept in view_node.columns:
            obj = {'id': concept.id, 'name': concept.name}
            if concept.id in ordering:
                obj['direction'] = ordering[concept.id]
            header.append(obj)

        objects = []
        for row in exporter.read(iterator):
            pk = None
            values = []
            for i, output in enumerate(row):
                if i == 0:
                    pk = output[pk_name]
                else:
                    values.extend(output.values())
            objects.append({'pk': pk, 'values': values})

        # Various other model options
        opts = queryset.model._meta
        model_name = opts.verbose_name.format()
        model_name_plural = opts.verbose_name_plural.format()

        data = {
            'keys': header,
            'objects': objects,
            'object_name': model_name,
            'object_name_plural': model_name_plural,
            'per_page': paginator.per_page,
            'num_pages': paginator.num_pages,
            'page_num': page.number,
        }

        # Augment previous and next page links if other pages exist
        links = {
            'self': {
                'href': uri(reverse('serrano:data:preview') + '?page=' +
                    str(page.number)),
            },
            'base': {
                'href': uri(reverse('serrano:data:preview')),
            }
        }

        if page.number != 1:
            links['prev'] = {
                'href': uri(reverse('serrano:data:preview') + '?page=' +
                    str(page.number - 1)),
            }

        if page.number < paginator.num_pages - 1:
            links['next'] = {
                'href': uri(reverse('serrano:data:preview') + '?page=' +
                    str(page.number + 1)),
            }

        data['_links'] = links

        return data

    post = get

preview_resource = PreviewResource()

# Resource endpoints
urlpatterns = patterns('',
    url(r'^$', preview_resource, name='preview'),
)
<|endoftext|>