<commit_before># -*- coding: utf-8 -*-
"""
    sc2reader
    ----------

    A Starcraft II replay parsing library intended to promote innovation in
    Starcraft tools and communities. Eventually, it will cover all official
    releases and dump easily to JSON for inter-language portabilty.

    sc2reader has been carefully written for clarity to serve as a starting
    point for those who want to write their own parsers, potentially in other
    languages for native access.

    Enjoy.

    :copyright: (c) 2011 Graylin Kim
    :license: MIT, See LICENSE.txt for details
"""

__version__ = '0.3-dev'

#System imports
import os

#PyPi imports
import mpyq

#Package imports
import config
import objects
import utils
import exceptions

class Reader(object):
    """ The SC2Reader class acts as a factory class for replay objects. The
        class accepts a key=value list of options to override defaults (see
        config.py) and exposes a very simple read/configure interface and
        orchestrates the replay build process.
    """

    def __init__(self, **user_options):
        """ The constructor makes a copy of the default_options to make sure the
            option configuration doesn't propogate back to the default_options.
            It should support any arbitrary number of different Reader objects.
        """
        self.options = config.default_options.copy()
        self.sys = utils.AttributeDict()
        self.configure(**user_options)

    def configure(self, **options):
        self.options.update(options)

        # Depending on the options choosen, the system needs to update related
        # options and setting in order to get the reading right.
        self.sys = config.full if self.options.parse_events else config.partial

    def read(self, location, **user_options):
        """ Read indicated file or recursively read matching files from the
            specified directory. Returns a replay or a list of replays depending
            on the context.
        """

        # Base the options off a copy to leave the Reader options uneffected.
        options = self.options.copy()
        options.update(user_options)

        # The directory option allows users to specify file locations relative
        # to a location other than the present working directory by joining the
        # location with the directory of their choice.
        if options.directory:
            location = os.path.join(options.directory, location)

        # When passed a directory as the location, the Reader recursively builds
        # a list of replays to return using the utils.get_files function. This
        # function respects the following arguments:
        #   * depth: The maximum depth to traverse. Defaults to unlimited (-1)
        #   * follow_symlinks: Boolean for following symlinks. Defaults to True
        #   * exclude_dirs: A list of directory names to skip while recursing
        #   * incldue_regex: A regular expression rule which all returned file
        #       names must match. Defaults to None
        #
        replays, files = list(), utils.get_files(location, **options)

        # If no files are found, it could be for a variety of reasons
        # raise a NoMatchingFilesError to alert them to the situation
        if not files:
            raise exceptions.NoMatchingFilesError()

        for location in files:
            if options.verbose: print "Reading: %s" % location

            with open(location) as replay_file:
                # The Replay constructor scans the header of the replay file for
                # the build number and stores the options for later use. The
                # options are copied so subsequent option changes are isolated.
                replay = objects.Replay(replay_file, **options.copy())

                # .SC2Replay files are written in Blizzard's MPQ Archive format.
                # The format stores a header which contains a block table that
                # specifies the location of each encrypted file.
                #
                # Unfortunately, some replay sites modify the replay contents to
                # add messages promoting their sites without updating the header
                # correctly. The listfile option(hack) lets us bypass this issue
                # by specifying the files we want instead of generating a list.
                archive = mpyq.MPQArchive(location, listfile=False)

                # These files are configured for either full or partial parsing
                for file in self.sys.files:

                    # For each file, we build a smart buffer object from the
                    # utf-8 encoded bitstream that mpyq extracts.
                    buffer = utils.ReplayBuffer(archive.read_file(file))

                    # Each version of Starcraft slightly modifies some portions
                    # of the format for some files. To work with this, the
                    # config file has a nested lookup structure of
                    # [build][file]=>reader which returns the appropriate reader
                    #
                    # TODO: Different versions also have different data mappings
                    #       sc2reader doesn't yet handle this difficulty.
                    #
                    # Readers use the type agnostic __call__ interface so that
                    # they can be implemented as functions or classes as needed.
                    #
                    # Readers return the extracted information from the buffer
                    # object which gets stored into the raw data dict for later
                    # use in post processing because correct interpretation of
                    # the information often requires data from other files.
                    reader = config.readers[replay.build][file]
                    reference_name = '_'.join(file.split('.')[1:])
                    replay.raw[reference_name] = reader(buffer, replay)

                # Now that the replay has been loaded with the "raw" data from
                # the archive files we run the system level post processors to
                # organize the data into a cross referenced data structure.
                #
                # After system level processors have run, call each of the post
                # processors provided by the user. This would be a good place to
                # convert the object to a serialized json string for cross
                # language processes or add custom attributes.
                #
                # TODO: Maybe we should switch this to a hook based architecture
                #       Needs to be able to load "contrib" type processors..
                for process in self.sys.processors+self.options.processors:
                    replay = process(replay)

                replays.append(replay)

        return replays

    def read_file(self, file, **options):
        replays = self.read(file, **options)

        # While normal usage would suggest passing in only filenames, it is
        # possible that directories could be passed in. Don't fail silently!
        if len(replays) > 1:
            raise exceptions.MultipleMatchingFilesError(replays)

        # Propogate the replay in a singular context
        return replays[0] if len(replays) > 0 else None

"""sc2reader uses a default SC2Reader class instance to provide a package level
interface to its functionality. The package level interface presents the same
functional interface, it just saves the hassel of creating the class object.
"""
__defaultReader = Reader()

def read(location, **user_options):
    return __defaultReader.read(location, **user_options)

def configure(**options):
    config.default_options.update(options)

def reset():
    __defaultReader = Reader()
<commit_msg>Add the read_file method on the sc2reader package.<commit_after># -*- coding: utf-8 -*-
"""
    sc2reader
    ----------

    A Starcraft II replay parsing library intended to promote innovation in
    Starcraft tools and communities. Eventually, it will cover all official
    releases and dump easily to JSON for inter-language portabilty.

    sc2reader has been carefully written for clarity to serve as a starting
    point for those who want to write their own parsers, potentially in other
    languages for native access.

    Enjoy.

    :copyright: (c) 2011 Graylin Kim
    :license: MIT, See LICENSE.txt for details
"""

__version__ = '0.3-dev'

#System imports
import os

#PyPi imports
import mpyq

#Package imports
import config
import objects
import utils
import exceptions

class Reader(object):
    """ The SC2Reader class acts as a factory class for replay objects. The
        class accepts a key=value list of options to override defaults (see
        config.py) and exposes a very simple read/configure interface and
        orchestrates the replay build process.
    """

    def __init__(self, **user_options):
        """ The constructor makes a copy of the default_options to make sure the
            option configuration doesn't propogate back to the default_options.
            It should support any arbitrary number of different Reader objects.
        """
        self.options = config.default_options.copy()
        self.sys = utils.AttributeDict()
        self.configure(**user_options)

    def configure(self, **options):
        self.options.update(options)

        # Depending on the options choosen, the system needs to update related
        # options and setting in order to get the reading right.
        self.sys = config.full if self.options.parse_events else config.partial

    def read(self, location, **user_options):
        """ Read indicated file or recursively read matching files from the
            specified directory. Returns a replay or a list of replays depending
            on the context.
        """

        # Base the options off a copy to leave the Reader options uneffected.
        options = self.options.copy()
        options.update(user_options)

        # The directory option allows users to specify file locations relative
        # to a location other than the present working directory by joining the
        # location with the directory of their choice.
        if options.directory:
            location = os.path.join(options.directory, location)

        # When passed a directory as the location, the Reader recursively builds
        # a list of replays to return using the utils.get_files function. This
        # function respects the following arguments:
        #   * depth: The maximum depth to traverse. Defaults to unlimited (-1)
        #   * follow_symlinks: Boolean for following symlinks. Defaults to True
        #   * exclude_dirs: A list of directory names to skip while recursing
        #   * incldue_regex: A regular expression rule which all returned file
        #       names must match. Defaults to None
        #
        replays, files = list(), utils.get_files(location, **options)

        # If no files are found, it could be for a variety of reasons
        # raise a NoMatchingFilesError to alert them to the situation
        if not files:
            raise exceptions.NoMatchingFilesError()

        for location in files:
            if options.verbose: print "Reading: %s" % location

            with open(location) as replay_file:
                # The Replay constructor scans the header of the replay file for
                # the build number and stores the options for later use. The
                # options are copied so subsequent option changes are isolated.
                replay = objects.Replay(replay_file, **options.copy())

                # .SC2Replay files are written in Blizzard's MPQ Archive format.
                # The format stores a header which contains a block table that
                # specifies the location of each encrypted file.
                #
                # Unfortunately, some replay sites modify the replay contents to
                # add messages promoting their sites without updating the header
                # correctly. The listfile option(hack) lets us bypass this issue
                # by specifying the files we want instead of generating a list.
                archive = mpyq.MPQArchive(location, listfile=False)

                # These files are configured for either full or partial parsing
                for file in self.sys.files:

                    # For each file, we build a smart buffer object from the
                    # utf-8 encoded bitstream that mpyq extracts.
                    buffer = utils.ReplayBuffer(archive.read_file(file))

                    # Each version of Starcraft slightly modifies some portions
                    # of the format for some files. To work with this, the
                    # config file has a nested lookup structure of
                    # [build][file]=>reader which returns the appropriate reader
                    #
                    # TODO: Different versions also have different data mappings
                    #       sc2reader doesn't yet handle this difficulty.
                    #
                    # Readers use the type agnostic __call__ interface so that
                    # they can be implemented as functions or classes as needed.
                    #
                    # Readers return the extracted information from the buffer
                    # object which gets stored into the raw data dict for later
                    # use in post processing because correct interpretation of
                    # the information often requires data from other files.
                    reader = config.readers[replay.build][file]
                    reference_name = '_'.join(file.split('.')[1:])
                    replay.raw[reference_name] = reader(buffer, replay)

                # Now that the replay has been loaded with the "raw" data from
                # the archive files we run the system level post processors to
                # organize the data into a cross referenced data structure.
                #
                # After system level processors have run, call each of the post
                # processors provided by the user. This would be a good place to
                # convert the object to a serialized json string for cross
                # language processes or add custom attributes.
                #
                # TODO: Maybe we should switch this to a hook based architecture
                #       Needs to be able to load "contrib" type processors..
                for process in self.sys.processors+self.options.processors:
                    replay = process(replay)

                replays.append(replay)

        return replays

    def read_file(self, file, **options):
        replays = self.read(file, **options)

        # While normal usage would suggest passing in only filenames, it is
        # possible that directories could be passed in. Don't fail silently!
        if len(replays) > 1:
            raise exceptions.MultipleMatchingFilesError(replays)

        # Propogate the replay in a singular context
        return replays[0] if len(replays) > 0 else None

"""sc2reader uses a default SC2Reader class instance to provide a package level
interface to its functionality. The package level interface presents the same
functional interface, it just saves the hassel of creating the class object.
"""
__defaultReader = Reader()

def read_file(location, **user_options):
    return __defaultReader.read_file(location, **user_options)

def read(location, **user_options):
    return __defaultReader.read(location, **user_options)

def configure(**options):
    config.default_options.update(options)

def reset():
    __defaultReader = Reader()
<|endoftext|>


<commit_before>from flask import Blueprint, request, jsonify, make_response
from app.{resources}.models import {Resources}, {Resources}Schema
from flask_restful import Api
from app.baseviews import Resource
from app.basemodels import db
from sqlalchemy.exc import SQLAlchemyError
from marshmallow import ValidationError

{resources} = Blueprint('{resources}', __name__)
# http://marshmallow.readthedocs.org/en/latest/quickstart.html#declaring-schemas
#https://github.com/marshmallow-code/marshmallow-jsonapi
schema = {Resources}Schema(strict=True)
api = Api({resources})

# {Resources}
class CreateList{Resources}(Resource):
    """http://jsonapi.org/format/#fetching
    A server MUST respond to a successful request to fetch an individual resource or resource collection with a 200 OK response.
    A server MUST respond with 404 Not Found when processing a request to fetch a single resource that does not exist, except when the request warrants a 200 OK response with null as the primary data (as described above)
    a self link as part of the top-level links object"""
    def get(self):
        {resources}_query = {Resources}.query.all()
        results = schema.dump({resources}_query, many=True).data
        return results

    """http://jsonapi.org/format/#crud
    A resource can be created by sending a POST request to a URL that represents a collection of {resources}. The request MUST include a single resource object as primary data. The resource object MUST contain at least a type member.
    If a POST request did not include a Client-Generated ID and the requested resource has been created successfully, the server MUST return a 201 Created status code"""

    def post(self):
        raw_dict = request.get_json(force=True)
        try:
                schema.validate(raw_dict)
                request_dict = raw_dict['data']['attributes']
                {resource} = {Resources}({add_fields})
                {resource}.add({resource})
                #Should not return password hash
                query = {Resources}.query.get({resource}.id)
                results = schema.dump(query).data
                return results, 201

        except ValidationError as err:
                resp = jsonify({{"error": err.messages}})
                resp.status_code = 403
                return resp

        except SQLAlchemyError as e:
                db.session.rollback()
                resp = jsonify({{"error": str(e)}})
                resp.status_code = 403
                return resp

class GetUpdateDelete{Resource}(Resource):

    """http://jsonapi.org/format/#fetching
    A server MUST respond to a successful request to fetch an individual resource or resource collection with a 200 OK response.
    A server MUST respond with 404 Not Found when processing a request to fetch a single resource that does not exist, except when the request warrants a 200 OK response with null as the primary data (as described above)
    a self link as part of the top-level links object"""

    def get(self, id):
        {resource}_query = {Resources}.query.get_or_404(id)
        result = schema.dump({resource}_query).data
        return result

    """http://jsonapi.org/format/#crud-updating"""

    def patch(self, id):
        {resource} = {Resources}.query.get_or_404(id)
        raw_dict = request.get_json(force=True)
        try:
            schema.validate(raw_dict)
            request_dict = raw_dict['data']['attributes']
            for key, value in request_dict.items():
                setattr({resource}, key, value)

            {resource}.update()
            return self.get(id)

        except ValidationError as err:
                resp = jsonify({{"error": err.messages}})
                resp.status_code = 401
                return resp

        except SQLAlchemyError as e:
                db.session.rollback()
                resp = jsonify({{"error": str(e)}})
                resp.status_code = 401
                return resp

    #http://jsonapi.org/format/#crud-deleting
    #A server MUST return a 204 No Content status code if a deletion request is successful and no content is returned.
    def delete(self, id):
        {resource} = {Resources}.query.get_or_404(id)
        try:
            delete = {resource}.delete({resource})
            response = make_response()
            response.status_code = 204
            return response

        except SQLAlchemyError as e:
                db.session.rollback()
                resp = jsonify({{"error": str(e)}})
                resp.status_code = 401
                return resp

api.add_resource(CreateList{Resources}, '.json')
api.add_resource(GetUpdateDelete{Resource}, '/<int:id>.json')<commit_msg> removed comment n made error str<commit_after>from flask import Blueprint, request, jsonify, make_response
from app.{resources}.models import {Resources}, {Resources}Schema
from flask_restful import Api
from app.baseviews import Resource
from app.basemodels import db
from sqlalchemy.exc import SQLAlchemyError
from marshmallow import ValidationError

{resources} = Blueprint('{resources}', __name__)
# http://marshmallow.readthedocs.org/en/latest/quickstart.html#declaring-schemas
#https://github.com/marshmallow-code/marshmallow-jsonapi
schema = {Resources}Schema(strict=True)
api = Api({resources})

# {Resources}
class CreateList{Resources}(Resource):
    """http://jsonapi.org/format/#fetching
    A server MUST respond to a successful request to fetch an individual resource or resource collection with a 200 OK response.
    A server MUST respond with 404 Not Found when processing a request to fetch a single resource that does not exist, except when the request warrants a 200 OK response with null as the primary data (as described above)
    a self link as part of the top-level links object"""
    def get(self):
        {resources}_query = {Resources}.query.all()
        results = schema.dump({resources}_query, many=True).data
        return results

    """http://jsonapi.org/format/#crud
    A resource can be created by sending a POST request to a URL that represents a collection of {resources}. The request MUST include a single resource object as primary data. The resource object MUST contain at least a type member.
    If a POST request did not include a Client-Generated ID and the requested resource has been created successfully, the server MUST return a 201 Created status code"""

    def post(self):
        raw_dict = request.get_json(force=True)
        try:
                schema.validate(raw_dict)
                request_dict = raw_dict['data']['attributes']
                {resource} = {Resources}({add_fields})
                {resource}.add({resource})
                query = {Resources}.query.get({resource}.id)
                results = schema.dump(query).data
                return results, 201

        except ValidationError as err:
                resp = jsonify({{"error": str(err.messages)}})
                resp.status_code = 403
                return resp

        except SQLAlchemyError as e:
                db.session.rollback()
                resp = jsonify({{"error": str(e)}})
                resp.status_code = 403
                return resp

class GetUpdateDelete{Resource}(Resource):

    """http://jsonapi.org/format/#fetching
    A server MUST respond to a successful request to fetch an individual resource or resource collection with a 200 OK response.
    A server MUST respond with 404 Not Found when processing a request to fetch a single resource that does not exist, except when the request warrants a 200 OK response with null as the primary data (as described above)
    a self link as part of the top-level links object"""

    def get(self, id):
        {resource}_query = {Resources}.query.get_or_404(id)
        result = schema.dump({resource}_query).data
        return result

    """http://jsonapi.org/format/#crud-updating"""

    def patch(self, id):
        {resource} = {Resources}.query.get_or_404(id)
        raw_dict = request.get_json(force=True)
        try:
            schema.validate(raw_dict)
            request_dict = raw_dict['data']['attributes']
            for key, value in request_dict.items():
                setattr({resource}, key, value)

            {resource}.update()
            return self.get(id)

        except ValidationError as err:
                resp = jsonify({{"error": str(err.messages)}})
                resp.status_code = 401
                return resp

        except SQLAlchemyError as e:
                db.session.rollback()
                resp = jsonify({{"error": str(e)}})
                resp.status_code = 401
                return resp

    #http://jsonapi.org/format/#crud-deleting
    #A server MUST return a 204 No Content status code if a deletion request is successful and no content is returned.
    def delete(self, id):
        {resource} = {Resources}.query.get_or_404(id)
        try:
            delete = {resource}.delete({resource})
            response = make_response()
            response.status_code = 204
            return response

        except SQLAlchemyError as e:
                db.session.rollback()
                resp = jsonify({{"error": str(e)}})
                resp.status_code = 401
                return resp

api.add_resource(CreateList{Resources}, '.json')
api.add_resource(GetUpdateDelete{Resource}, '/<int:id>.json')<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Type coders registration.

This module contains functionality to define and use coders for custom classes.
Let's say we have a class Xyz and we are processing a PCollection with elements
of type Xyz. If we do not register a coder for Xyz, a default pickle-based
fallback coder will be used. This can be undesirable for two reasons. First, we
may want a faster coder or a more space efficient one. Second, the pickle-based
coder is not deterministic in the sense that objects like dictionaries or sets
are not guaranteed to be encoded in the same way every time (elements are not
really ordered).

Two (sometimes three) steps are needed to define and use a custom coder:
  - define the coder class
  - associate the code with the class (a.k.a. coder registration)
  - typehint DoFns or transforms with the new class or composite types using
    the class.

A coder class is defined by subclassing from CoderBase and defining the
encode_to_bytes and decode_from_bytes methods. The framework uses duck-typing
for coders so it is not strictly required to subclass from CoderBase as long as
the encode/decode methods are defined.

Registering a coder class is made with a register_coder() call::

  from apache_beam import coders
  ...
  coders.registry.register_coder(Xyz, XyzCoder)

Additionally, DoFns and PTransforms may need type hints. This is not always
necessary since there is functionality to infer the return types of DoFns by
analyzing the code. For instance, for the function below the return type of
'Xyz' will be inferred::

  def MakeXyzs(v):
    return Xyz(v)

If Xyz is inferred then its coder will be used whenever the framework needs to
serialize data (e.g., writing to the shuffler subsystem responsible for group by
key operations). If a typehint is needed it can be specified by decorating the
DoFns or using with_input_types/with_output_types methods on PTransforms. For
example, the above function can be decorated::

  @with_output_types(Xyz)
  def MakeXyzs(v):
    return complex_operation_returning_Xyz(v)

See apache_beam.typehints.decorators module for more details.
"""

import logging

from apache_beam.coders import coders
from apache_beam.typehints import typehints

class CoderRegistry(object):
  """A coder registry for typehint/coder associations."""

  def __init__(self, fallback_coder=None):
    self._coders = {}
    self.custom_types = []
    self.register_standard_coders(fallback_coder)

  def register_standard_coders(self, fallback_coder):
    """Register coders for all basic and composite types."""
    self._register_coder_internal(int, coders.VarIntCoder)
    self._register_coder_internal(float, coders.FloatCoder)
    self._register_coder_internal(str, coders.BytesCoder)
    self._register_coder_internal(bytes, coders.BytesCoder)
    self._register_coder_internal(unicode, coders.StrUtf8Coder)
    self._register_coder_internal(typehints.TupleConstraint, coders.TupleCoder)
    # Default fallback coders applied in that order until the first matching
    # coder found.
    default_fallback_coders = [coders.ProtoCoder, coders.FastPrimitivesCoder]
    self._fallback_coder = fallback_coder or FirstOf(default_fallback_coders)

  def _register_coder_internal(self, typehint_type, typehint_coder_class):
    self._coders[typehint_type] = typehint_coder_class

  def register_coder(self, typehint_type, typehint_coder_class):
    if not isinstance(typehint_coder_class, type):
      raise TypeError('Coder registration requires a coder class object. '
                      'Received %r instead.' % typehint_coder_class)
    if typehint_type not in self.custom_types:
      self.custom_types.append(typehint_type)
    self._register_coder_internal(typehint_type, typehint_coder_class)

  def get_coder(self, typehint):
    coder = self._coders.get(
        typehint.__class__ if isinstance(typehint, typehints.TypeConstraint)
        else typehint, None)
    if isinstance(typehint, typehints.TypeConstraint) and coder is not None:
      return coder.from_type_hint(typehint, self)
    if coder is None:
      # We use the fallback coder when there is no coder registered for a
      # typehint. For example a user defined class with no coder specified.
      if not hasattr(self, '_fallback_coder'):
        raise RuntimeError(
            'Coder registry has no fallback coder. This can happen if the '
            'fast_coders module could not be imported.')
      if isinstance(typehint, typehints.IterableTypeConstraint):
        # In this case, we suppress the warning message for using the fallback
        # coder, since Iterable is hinted as the output of a GroupByKey
        # operation and that direct output will not be coded.
        # TODO(ccy): refine this behavior.
        pass
      elif typehint is None:
        # In some old code, None is used for Any.
        # TODO(robertwb): Clean this up.
        pass
      elif typehint is object:
        # We explicitly want the fallback coder.
        pass
      elif isinstance(typehint, typehints.TypeVariable):
        # TODO(robertwb): Clean this up when type inference is fully enabled.
        pass
      else:
        logging.warning('Using fallback coder for typehint: %r.', typehint)
      coder = self._fallback_coder
    return coder.from_type_hint(typehint, self)

  def get_custom_type_coder_tuples(self, types):
    """Returns type/coder tuples for all custom types passed in."""
    return [(t, self._coders[t]) for t in types if t in self.custom_types]

  def verify_deterministic(self, key_coder, op_name, silent=True):
    if not key_coder.is_deterministic():
      error_msg = ('The key coder "%s" for %s '
                   'is not deterministic. This may result in incorrect '
                   'pipeline output. This can be fixed by adding a type '
                   'hint to the operation preceding the GroupByKey step, '
                   'and for custom key classes, by writing a '
                   'deterministic custom Coder. Please see the '
                   'documentation for more details.' % (key_coder, op_name))
      # TODO(vikasrk): PickleCoder will eventually be removed once its direct
      # usage is stopped.
      if isinstance(key_coder, (coders.PickleCoder,
                                coders.FastPrimitivesCoder)):
        if not silent:
          logging.warning(error_msg)
        return coders.DeterministicFastPrimitivesCoder(key_coder, op_name)
      else:
        raise ValueError(error_msg)
    else:
      return key_coder

class FirstOf(object):
  "A class used to get the first matching coder from a list of coders."

  def __init__(self, coders):
    self._coders = coders

  def from_type_hint(self, typehint, registry):
    messages = []
    for coder in self._coders:
      try:
        return coder.from_type_hint(typehint, self)
      except Exception as e:
        msg = ('%s could not provide a Coder for type %s: %s' %
               (coder, typehint, e))
        messages.append(msg)

    raise ValueError('Cannot provide coder for %s: %s' %
                     (typehint, ';'.join(messages)))

registry = CoderRegistry()
<commit_msg>Closes #1754<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Type coders registration.

This module contains functionality to define and use coders for custom classes.
Let's say we have a class Xyz and we are processing a PCollection with elements
of type Xyz. If we do not register a coder for Xyz, a default pickle-based
fallback coder will be used. This can be undesirable for two reasons. First, we
may want a faster coder or a more space efficient one. Second, the pickle-based
coder is not deterministic in the sense that objects like dictionaries or sets
are not guaranteed to be encoded in the same way every time (elements are not
really ordered).

Two (sometimes three) steps are needed to define and use a custom coder:
  - define the coder class
  - associate the code with the class (a.k.a. coder registration)
  - typehint DoFns or transforms with the new class or composite types using
    the class.

A coder class is defined by subclassing from CoderBase and defining the
encode_to_bytes and decode_from_bytes methods. The framework uses duck-typing
for coders so it is not strictly required to subclass from CoderBase as long as
the encode/decode methods are defined.

Registering a coder class is made with a register_coder() call::

  from apache_beam import coders
  ...
  coders.registry.register_coder(Xyz, XyzCoder)

Additionally, DoFns and PTransforms may need type hints. This is not always
necessary since there is functionality to infer the return types of DoFns by
analyzing the code. For instance, for the function below the return type of
'Xyz' will be inferred::

  def MakeXyzs(v):
    return Xyz(v)

If Xyz is inferred then its coder will be used whenever the framework needs to
serialize data (e.g., writing to the shuffler subsystem responsible for group by
key operations). If a typehint is needed it can be specified by decorating the
DoFns or using with_input_types/with_output_types methods on PTransforms. For
example, the above function can be decorated::

  @with_output_types(Xyz)
  def MakeXyzs(v):
    return complex_operation_returning_Xyz(v)

See apache_beam.typehints.decorators module for more details.
"""

import logging
import warnings

from apache_beam.coders import coders
from apache_beam.typehints import typehints

class CoderRegistry(object):
  """A coder registry for typehint/coder associations."""

  def __init__(self, fallback_coder=None):
    self._coders = {}
    self.custom_types = []
    self.register_standard_coders(fallback_coder)

  def register_standard_coders(self, fallback_coder):
    """Register coders for all basic and composite types."""
    self._register_coder_internal(int, coders.VarIntCoder)
    self._register_coder_internal(float, coders.FloatCoder)
    self._register_coder_internal(str, coders.BytesCoder)
    self._register_coder_internal(bytes, coders.BytesCoder)
    self._register_coder_internal(unicode, coders.StrUtf8Coder)
    self._register_coder_internal(typehints.TupleConstraint, coders.TupleCoder)
    # Default fallback coders applied in that order until the first matching
    # coder found.
    default_fallback_coders = [coders.ProtoCoder, coders.FastPrimitivesCoder]
    self._fallback_coder = fallback_coder or FirstOf(default_fallback_coders)

  def _register_coder_internal(self, typehint_type, typehint_coder_class):
    self._coders[typehint_type] = typehint_coder_class

  def register_coder(self, typehint_type, typehint_coder_class):
    if not isinstance(typehint_coder_class, type):
      raise TypeError('Coder registration requires a coder class object. '
                      'Received %r instead.' % typehint_coder_class)
    if typehint_type not in self.custom_types:
      self.custom_types.append(typehint_type)
    self._register_coder_internal(typehint_type, typehint_coder_class)

  def get_coder(self, typehint):
    coder = self._coders.get(
        typehint.__class__ if isinstance(typehint, typehints.TypeConstraint)
        else typehint, None)
    if isinstance(typehint, typehints.TypeConstraint) and coder is not None:
      return coder.from_type_hint(typehint, self)
    if coder is None:
      # We use the fallback coder when there is no coder registered for a
      # typehint. For example a user defined class with no coder specified.
      if not hasattr(self, '_fallback_coder'):
        raise RuntimeError(
            'Coder registry has no fallback coder. This can happen if the '
            'fast_coders module could not be imported.')
      if isinstance(typehint, typehints.IterableTypeConstraint):
        # In this case, we suppress the warning message for using the fallback
        # coder, since Iterable is hinted as the output of a GroupByKey
        # operation and that direct output will not be coded.
        # TODO(ccy): refine this behavior.
        pass
      elif typehint is None:
        # In some old code, None is used for Any.
        # TODO(robertwb): Clean this up.
        pass
      elif typehint is object:
        # We explicitly want the fallback coder.
        pass
      elif isinstance(typehint, typehints.TypeVariable):
        # TODO(robertwb): Clean this up when type inference is fully enabled.
        pass
      else:
        warnings.warn('Using fallback coder for typehint: %r.' % typehint)
      coder = self._fallback_coder
    return coder.from_type_hint(typehint, self)

  def get_custom_type_coder_tuples(self, types):
    """Returns type/coder tuples for all custom types passed in."""
    return [(t, self._coders[t]) for t in types if t in self.custom_types]

  def verify_deterministic(self, key_coder, op_name, silent=True):
    if not key_coder.is_deterministic():
      error_msg = ('The key coder "%s" for %s '
                   'is not deterministic. This may result in incorrect '
                   'pipeline output. This can be fixed by adding a type '
                   'hint to the operation preceding the GroupByKey step, '
                   'and for custom key classes, by writing a '
                   'deterministic custom Coder. Please see the '
                   'documentation for more details.' % (key_coder, op_name))
      # TODO(vikasrk): PickleCoder will eventually be removed once its direct
      # usage is stopped.
      if isinstance(key_coder, (coders.PickleCoder,
                                coders.FastPrimitivesCoder)):
        if not silent:
          logging.warning(error_msg)
        return coders.DeterministicFastPrimitivesCoder(key_coder, op_name)
      else:
        raise ValueError(error_msg)
    else:
      return key_coder

class FirstOf(object):
  "A class used to get the first matching coder from a list of coders."

  def __init__(self, coders):
    self._coders = coders

  def from_type_hint(self, typehint, registry):
    messages = []
    for coder in self._coders:
      try:
        return coder.from_type_hint(typehint, self)
      except Exception as e:
        msg = ('%s could not provide a Coder for type %s: %s' %
               (coder, typehint, e))
        messages.append(msg)

    raise ValueError('Cannot provide coder for %s: %s' %
                     (typehint, ';'.join(messages)))

registry = CoderRegistry()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import boto
from numbers import Number
from boto.dynamodb.exceptions import DynamoDBKeyNotFoundError

class Table(object):
    def __init__(self, table=None, eager=False):
        self.table = table
        self.is_eager = eager

    def __repr__(self):
        return '<table \'{0}\'>'.format(self.name)

    @property
    def name(self):
        return self.table.__dict__['_dict']['TableName']

    def item(self, item):
        return Item(item, self)

    def delete(self):
        return self.table.delete()

    def scale(self, read=None, write=None):
        read = read or self.table.read_units
        write = write or self.table.read_units
        return self.table.update_throughput(read_units=read, write_units=write)

    def __getitem__(self, key):
        try:
            if isinstance(key, (basestring, Number)):
                key = [key]
            i = self.table.get_item(*key)
            i = self.item(i)
        except DynamoDBKeyNotFoundError:
            return self.__magic_get(key)
        return i

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __setitem__(self, key, values):
        if isinstance(key, (basestring, Number)):
            key = [key]
        i = self.table.new_item(*key, attrs=values)
        i = self.item(i)
        i.put()
        return i

    def __delitem__(self, key):
        return self[key].delete()

    def __magic_get(self, key):
        if self.is_eager:
            self[key] = {}
            return self.item(self[key])

    def __contains__(self, key):
        return not self.get(key) is None

    def new(self, name):
        table = self.table.layer2.create_table(
            name=name,
            schema=self.table._schema,
            read_units=self.table.read_units,
            write_units=self.table.write_units
        )
        return Table(table=table, eager=self.is_eager)

class Item(object):
    def __init__(self, item, table):
        self.item = item
        self.table = table

    @property
    def is_eager(self):
        return self.table.is_eager

    def __getattr__(self, key):
        if not key in ['item']:
            try:
                return getattr(object, key)
            except AttributeError:
                return getattr(self.item, key)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __repr__(self):
        return repr(self.item)

    def __getitem__(self, key):
        return self.item[key]

    def __setitem__(self, key, value):
        self.item[key] = value
        if self.is_eager:
            self.item.save()

    def __contains__(self, key):
        return key in self.item

def table(name, auth=None, eager=True):
    """Returns a given table for the given user."""
    if auth:
        dynamodb = boto.connect_dynamodb(*auth)
    else:
        dynamodb = boto.connect_dynamodb()

    table = dynamodb.get_table(name)
    return Table(table=table, eager=eager)

def tables(auth=None, eager=True):
    """Returns a list of tables for the given user."""
    if auth:
        dynamodb = boto.connect_dynamodb(*auth)
    else:
        dynamodb = boto.connect_dynamodb()

    return [table(t, auth, eager=eager) for t in dynamodb.list_tables()]
<commit_msg>Cleaning up that nastiness.<commit_after># -*- coding: utf-8 -*-

import boto
from numbers import Number
from boto.dynamodb.exceptions import DynamoDBKeyNotFoundError

class Table(object):
    def __init__(self, table=None, eager=False):
        self.table = table
        self.is_eager = eager

    def __repr__(self):
        return '<table \'{0}\'>'.format(self.name)

    @property
    def name(self):
        return self.table.__dict__['_dict']['TableName']

    def item(self, item):
        return Item(item, self)

    def delete(self):
        return self.table.delete()

    def scale(self, read=None, write=None):
        read = read or self.table.read_units
        write = write or self.table.read_units
        return self.table.update_throughput(read_units=read, write_units=write)

    def __getitem__(self, key):
        try:
            if isinstance(key, (basestring, Number)):
                key = [key]
            i = self.table.get_item(*key)
            i = self.item(i)
        except DynamoDBKeyNotFoundError:
            return self.__magic_get(key)
        return i

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __setitem__(self, key, values):
        if isinstance(key, (basestring, Number)):
            key = [key]
        i = self.table.new_item(*key, attrs=values)
        i = self.item(i)
        i.put()
        return i

    def __delitem__(self, key):
        return self[key].delete()

    def __magic_get(self, key):
        if self.is_eager:
            self[key] = {}
            return self.item(self[key])

    def __contains__(self, key):
        return not self.get(key) is None

    def new(self, name):
        table = self.table.layer2.create_table(
            name=name,
            schema=self.table._schema,
            read_units=self.table.read_units,
            write_units=self.table.write_units
        )
        return Table(table=table, eager=self.is_eager)

class Item(object):
    def __init__(self, item, table):
        self.item = item
        self.table = table

    @property
    def is_eager(self):
        return self.table.is_eager

    def __getattr__(self, key):
        if not key in ['item']:
            try:
                return getattr(object, key)
            except AttributeError:
                return getattr(self.item, key)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __repr__(self):
        return repr(self.item)

    def __getitem__(self, key):
        return self.item[key]

    def __setitem__(self, key, value):
        self.item[key] = value
        if self.is_eager:
            self.item.save()

    def __contains__(self, key):
        return key in self.item

def table(name, auth=None, eager=True):
    """Returns a given table for the given user."""
    auth = auth or []
    dynamodb = boto.connect_dynamodb(*auth)

    table = dynamodb.get_table(name)
    return Table(table=table, eager=eager)

def tables(auth=None, eager=True):
    """Returns a list of tables for the given user."""
    auth = auth or []
    dynamodb = boto.connect_dynamodb(*auth)

    return [table(t, auth, eager=eager) for t in dynamodb.list_tables()]
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
This module is for internal use only; no backwards-compatibility guarantees.

The classes in this file keep shared state, and organize metrics information.

Available classes:

- MetricKey - Internal key for a metric.
- MetricResult - Current status of a metric's updates/commits.
- _MetricsEnvironment - Keeps track of MetricsContainer and other metrics
    information for every single execution working thread.
- MetricsContainer - Holds the metrics of a single step and a single
    unit-of-commit (bundle).
"""

from __future__ import absolute_import

import threading
from builtins import object
from collections import defaultdict

from apache_beam.metrics.cells import CounterCell
from apache_beam.metrics.cells import DistributionCell
from apache_beam.metrics.cells import GaugeCell
from apache_beam.portability.api import beam_fn_api_pb2
from apache_beam.runners.worker import statesampler

class MetricKey(object):
  """Key used to identify instance of metric cell.

  Metrics are internally keyed by the step name they associated with and
  the name of the metric.
  """
  def __init__(self, step, metric):
    """Initializes ``MetricKey``.

    Args:
      step: A string with the step this metric cell is part of.
      metric: A ``MetricName`` that identifies a metric.
    """
    self.step = step
    self.metric = metric

  def __eq__(self, other):
    return (self.step == other.step and
            self.metric == other.metric)

  def __hash__(self):
    return hash((self.step, self.metric))

  def __repr__(self):
    return 'MetricKey(step={}, metric={})'.format(
        self.step, self.metric)

  def __hash__(self):
    return hash((self.step, self.metric))

class MetricResult(object):
  """Keeps track of the status of a metric within a single bundle.

  It contains the physical and logical updates to the metric. Physical updates
  are updates that have not necessarily been committed, but that have been made
  during pipeline execution. Logical updates are updates that have been
  committed.

  Attributes:
    key: A ``MetricKey`` that identifies the metric and bundle of this result.
    committed: The committed updates of the metric. This attribute's type is
      of metric type result (e.g. int, DistributionResult, GaugeResult).
    attempted: The logical updates of the metric. This attribute's type is that
      of metric type result (e.g. int, DistributionResult, GaugeResult).
  """
  def __init__(self, key, committed, attempted):
    """Initializes ``MetricResult``.
    Args:
      key: A ``MetricKey`` object.
      committed: Metric data that has been committed (e.g. logical updates)
      attempted: Metric data that has been attempted (e.g. physical updates)
    """
    self.key = key
    self.committed = committed
    self.attempted = attempted

  def __eq__(self, other):
    return (self.key == other.key and
            self.committed == other.committed and
            self.attempted == other.attempted)

  def __hash__(self):
    return hash((self.key, self.committed, self.attempted))

  def __repr__(self):
    return 'MetricResult(key={}, committed={}, attempted={})'.format(
        self.key, str(self.committed), str(self.attempted))

class _MetricsEnvironment(object):
  """Holds the MetricsContainer for every thread and other metric information.

  This class is not meant to be instantiated, instead being used to keep
  track of global state.
  """
  def __init__(self):
    self.METRICS_SUPPORTED = False
    self._METRICS_SUPPORTED_LOCK = threading.Lock()
    self.PER_THREAD = threading.local()
    self.set_container_stack()

  def set_container_stack(self):
    if not hasattr(self.PER_THREAD, 'container'):
      self.PER_THREAD.container = []

  def container_stack(self):
    self.set_container_stack()
    return self.PER_THREAD.container

  def set_metrics_supported(self, supported):
    self.set_container_stack()
    with self._METRICS_SUPPORTED_LOCK:
      self.METRICS_SUPPORTED = supported

  def _old_style_container(self):
    """Gets the current MetricsContainer based on the container stack.

    The container stack is the old method, and will be deprecated. Should
    rely on StateSampler instead."""
    self.set_container_stack()
    index = len(self.PER_THREAD.container) - 1
    if index < 0:
      return None
    return self.PER_THREAD.container[index]

  def current_container(self):
    """Returns the current MetricsContainer."""
    sampler = statesampler.get_current_tracker()
    if sampler is None:
      return self._old_style_container()
    return sampler.current_state().metrics_container

MetricsEnvironment = _MetricsEnvironment()

class MetricsContainer(object):
  """Holds the metrics of a single step and a single bundle."""
  def __init__(self, step_name):
    self.step_name = step_name
    self.counters = defaultdict(lambda: CounterCell())
    self.distributions = defaultdict(lambda: DistributionCell())
    self.gauges = defaultdict(lambda: GaugeCell())

  def get_counter(self, metric_name):
    return self.counters[metric_name]

  def get_distribution(self, metric_name):
    return self.distributions[metric_name]

  def get_gauge(self, metric_name):
    return self.gauges[metric_name]

  def _get_updates(self, filter=None):
    """Return cumulative values of metrics filtered according to a lambda.

    This returns all the cumulative values for all metrics after filtering
    then with the filter parameter lambda function. If None is passed in,
    then cumulative values for all metrics are returned.
    """
    if filter is None:
      filter = lambda v: True
    counters = {MetricKey(self.step_name, k): v.get_cumulative()
                for k, v in self.counters.items()
                if filter(v)}

    distributions = {MetricKey(self.step_name, k): v.get_cumulative()
                     for k, v in self.distributions.items()
                     if filter(v)}

    gauges = {MetricKey(self.step_name, k): v.get_cumulative()
              for k, v in self.gauges.items()
              if filter(v)}

    return MetricUpdates(counters, distributions, gauges)

  def get_updates(self):
    """Return cumulative values of metrics that changed since the last commit.

    This returns all the cumulative values for all metrics only if their state
    prior to the function call was COMMITTING or DIRTY.
    """
    return self._get_updates(filter=lambda v: v.commit.before_commit())

  def get_cumulative(self):
    """Return MetricUpdates with cumulative values of all metrics in container.

    This returns all the cumulative values for all metrics regardless of whether
    they have been committed or not.
    """
    return self._get_updates()

  def to_runner_api(self):
    return (
        [beam_fn_api_pb2.Metrics.User(
            metric_name=k.to_runner_api(),
            counter_data=beam_fn_api_pb2.Metrics.User.CounterData(
                value=v.get_cumulative()))
         for k, v in self.counters.items()] +
        [beam_fn_api_pb2.Metrics.User(
            metric_name=k.to_runner_api(),
            distribution_data=v.get_cumulative().to_runner_api())
         for k, v in self.distributions.items()] +
        [beam_fn_api_pb2.Metrics.User(
            metric_name=k.to_runner_api(),
            gauge_data=v.get_cumulative().to_runner_api())
         for k, v in self.gauges.items()]
    )

class MetricUpdates(object):
  """Contains updates for several metrics.

  A metric update is an object containing information to update a metric.
  For Distribution metrics, it is DistributionData, and for Counter metrics,
  it's an int.
  """
  def __init__(self, counters=None, distributions=None, gauges=None):
    """Create a MetricUpdates object.

    Args:
      counters: Dictionary of MetricKey:MetricUpdate updates.
      distributions: Dictionary of MetricKey:MetricUpdate objects.
      gauges: Dictionary of MetricKey:MetricUpdate objects.
    """
    self.counters = counters or {}
    self.distributions = distributions or {}
    self.gauges = gauges or {}
<commit_msg>Remove old style metrics context management<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
This module is for internal use only; no backwards-compatibility guarantees.

The classes in this file keep shared state, and organize metrics information.

Available classes:

- MetricKey - Internal key for a metric.
- MetricResult - Current status of a metric's updates/commits.
- _MetricsEnvironment - Keeps track of MetricsContainer and other metrics
    information for every single execution working thread.
- MetricsContainer - Holds the metrics of a single step and a single
    unit-of-commit (bundle).
"""

from __future__ import absolute_import

import threading
from builtins import object
from collections import defaultdict

from apache_beam.metrics.cells import CounterCell
from apache_beam.metrics.cells import DistributionCell
from apache_beam.metrics.cells import GaugeCell
from apache_beam.portability.api import beam_fn_api_pb2
from apache_beam.runners.worker import statesampler

class MetricKey(object):
  """Key used to identify instance of metric cell.

  Metrics are internally keyed by the step name they associated with and
  the name of the metric.
  """
  def __init__(self, step, metric):
    """Initializes ``MetricKey``.

    Args:
      step: A string with the step this metric cell is part of.
      metric: A ``MetricName`` that identifies a metric.
    """
    self.step = step
    self.metric = metric

  def __eq__(self, other):
    return (self.step == other.step and
            self.metric == other.metric)

  def __hash__(self):
    return hash((self.step, self.metric))

  def __repr__(self):
    return 'MetricKey(step={}, metric={})'.format(
        self.step, self.metric)

  def __hash__(self):
    return hash((self.step, self.metric))

class MetricResult(object):
  """Keeps track of the status of a metric within a single bundle.

  It contains the physical and logical updates to the metric. Physical updates
  are updates that have not necessarily been committed, but that have been made
  during pipeline execution. Logical updates are updates that have been
  committed.

  Attributes:
    key: A ``MetricKey`` that identifies the metric and bundle of this result.
    committed: The committed updates of the metric. This attribute's type is
      of metric type result (e.g. int, DistributionResult, GaugeResult).
    attempted: The logical updates of the metric. This attribute's type is that
      of metric type result (e.g. int, DistributionResult, GaugeResult).
  """
  def __init__(self, key, committed, attempted):
    """Initializes ``MetricResult``.
    Args:
      key: A ``MetricKey`` object.
      committed: Metric data that has been committed (e.g. logical updates)
      attempted: Metric data that has been attempted (e.g. physical updates)
    """
    self.key = key
    self.committed = committed
    self.attempted = attempted

  def __eq__(self, other):
    return (self.key == other.key and
            self.committed == other.committed and
            self.attempted == other.attempted)

  def __hash__(self):
    return hash((self.key, self.committed, self.attempted))

  def __repr__(self):
    return 'MetricResult(key={}, committed={}, attempted={})'.format(
        self.key, str(self.committed), str(self.attempted))

class _MetricsEnvironment(object):
  """Holds the MetricsContainer for every thread and other metric information.

  This class is not meant to be instantiated, instead being used to keep
  track of global state.
  """
  def __init__(self):
    self.METRICS_SUPPORTED = False
    self._METRICS_SUPPORTED_LOCK = threading.Lock()

  def set_metrics_supported(self, supported):
    with self._METRICS_SUPPORTED_LOCK:
      self.METRICS_SUPPORTED = supported

  def current_container(self):
    """Returns the current MetricsContainer."""
    sampler = statesampler.get_current_tracker()
    if sampler is None:
      return None
    return sampler.current_state().metrics_container

MetricsEnvironment = _MetricsEnvironment()

class MetricsContainer(object):
  """Holds the metrics of a single step and a single bundle."""
  def __init__(self, step_name):
    self.step_name = step_name
    self.counters = defaultdict(lambda: CounterCell())
    self.distributions = defaultdict(lambda: DistributionCell())
    self.gauges = defaultdict(lambda: GaugeCell())

  def get_counter(self, metric_name):
    return self.counters[metric_name]

  def get_distribution(self, metric_name):
    return self.distributions[metric_name]

  def get_gauge(self, metric_name):
    return self.gauges[metric_name]

  def _get_updates(self, filter=None):
    """Return cumulative values of metrics filtered according to a lambda.

    This returns all the cumulative values for all metrics after filtering
    then with the filter parameter lambda function. If None is passed in,
    then cumulative values for all metrics are returned.
    """
    if filter is None:
      filter = lambda v: True
    counters = {MetricKey(self.step_name, k): v.get_cumulative()
                for k, v in self.counters.items()
                if filter(v)}

    distributions = {MetricKey(self.step_name, k): v.get_cumulative()
                     for k, v in self.distributions.items()
                     if filter(v)}

    gauges = {MetricKey(self.step_name, k): v.get_cumulative()
              for k, v in self.gauges.items()
              if filter(v)}

    return MetricUpdates(counters, distributions, gauges)

  def get_updates(self):
    """Return cumulative values of metrics that changed since the last commit.

    This returns all the cumulative values for all metrics only if their state
    prior to the function call was COMMITTING or DIRTY.
    """
    return self._get_updates(filter=lambda v: v.commit.before_commit())

  def get_cumulative(self):
    """Return MetricUpdates with cumulative values of all metrics in container.

    This returns all the cumulative values for all metrics regardless of whether
    they have been committed or not.
    """
    return self._get_updates()

  def to_runner_api(self):
    return (
        [beam_fn_api_pb2.Metrics.User(
            metric_name=k.to_runner_api(),
            counter_data=beam_fn_api_pb2.Metrics.User.CounterData(
                value=v.get_cumulative()))
         for k, v in self.counters.items()] +
        [beam_fn_api_pb2.Metrics.User(
            metric_name=k.to_runner_api(),
            distribution_data=v.get_cumulative().to_runner_api())
         for k, v in self.distributions.items()] +
        [beam_fn_api_pb2.Metrics.User(
            metric_name=k.to_runner_api(),
            gauge_data=v.get_cumulative().to_runner_api())
         for k, v in self.gauges.items()]
    )

class MetricUpdates(object):
  """Contains updates for several metrics.

  A metric update is an object containing information to update a metric.
  For Distribution metrics, it is DistributionData, and for Counter metrics,
  it's an int.
  """
  def __init__(self, counters=None, distributions=None, gauges=None):
    """Create a MetricUpdates object.

    Args:
      counters: Dictionary of MetricKey:MetricUpdate updates.
      distributions: Dictionary of MetricKey:MetricUpdate objects.
      gauges: Dictionary of MetricKey:MetricUpdate objects.
    """
    self.counters = counters or {}
    self.distributions = distributions or {}
    self.gauges = gauges or {}
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rospy
from sensor_msgs.msg import String

class StatusPrinter:

	def __init__(self):
		self.activity = "Idle"

	def print_activity(self):
		print " [ Activity  : " + activity + " "*len(activity-10) + " ]"

	def display_status(self):
		print_activity()

	def data_received(self, data):
		self.activity = data

		self.display_status()

def callback(data):
    rospy.loginfo(rospy.get_caller_id()+"I heard %s",data.data)

def listener():

    rospy.init_node('listener', anonymous=True)

    rospy.Subscriber("chatter", String, callback)

    # spin() simply keeps python from exiting until this node is stopped
    rospy.spin()

if __name__ == '__main__':
	rospy.init_node('status')

	rospy.Subscriber('', String, callback)

    listener()

<commit_msg>Deleted obsolete status_printer.py file.<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Convert a rosbag file to legacy lidar binary format.
"""

"""LIDAR datatype format is:
    (
        timestamp (long),
        flag (bool saved as int),
        accelerometer[3] (double),
        gps[3] (double),
        distance[LIDAR_NUM_ANGLES] (long),
    )

    'int' and 'long' are the same size on the raspberry pi (32 bits).
"""
import sys
import logging

# suppress logging warnings due to rospy
logging.basicConfig(filename='/dev/null')
import rosbag

def print_bag(bag):
    topics = ['/scan', '/flagbutton_pressed']
    for message in bag.read_messages(topics=topics):
        print(message)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print(('Usage: {} <rosbag> [<outfile>] \n\n'
               'Print contents of rosbag file. If <outfile> is provided, \n'
               'write contents of rosbag file to <outfile> in the legacy \n'
               'lidar binary format.').format(__file__))
        sys.exit(1)

    outfile = None
    filename = sys.argv[1]

    if len(sys.argv) == 3:
        # convert to outfile if provided
        outfile = sys.argv[2]

    with rosbag.Bag(filename) as bag:
        print_bag(bag)

    sys.exit()

<commit_msg>Add option to convert bagfile to lidar dtype<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Convert a rosbag file to legacy lidar binary format LIDAR_CONVERTED_DTYPE.
"""

import sys
import logging

import numpy as np

# suppress logging warnings due to rospy
logging.basicConfig(filename='/dev/null')
import rosbag

from antlia.dtype import LIDAR_CONVERTED_DTYPE

def print_bag(bag, topics=None):
    if topics is None:
        topics = ['/scan', '/flagbutton_pressed']

    for message in bag.read_messages(topics=topics):
        print(message)

def convert_bag(bag):
    topics = ['/scan', '/flagbutton_pressed']

    # assume button always starts as false
    button_state = False

    last_timestamp = 0
    record = []
    for topic, msg, t in bag.read_messages(topics=topics):

        # check that messages are monotonically increasing in time
        timestamp = msg.header.stamp.secs + 1e-9*msg.header.stamp.nsecs
        assert_msg = ('messages are not monotonically increasing in time\n'
                      'last_timestamp: {}\n'
                      'timestamp: {}\n').format(last_timestamp, timestamp)
        assert timestamp >= last_timestamp, assert_msg

        if topic == '/flagbutton_pressed':
            # use zero-order hold for button state
            button_state = msg.data
        elif topic == '/scan':
            # write datatype object for every lidar scan
            datum = np.recarray((1,), dtype=LIDAR_CONVERTED_DTYPE)

            # imu accelerometer and gps is ignored
            datum.time = timestamp
            datum.sync = button_state
            datum.distance = msg.ranges

            record.append(datum)
        else:
            raise KeyError('Unhandled topic: {}'.format(topic))

    return np.concatenate(record)

if __name__ == '__main__':
    OUTFILE_EXT = '.pkl.gz'

    if len(sys.argv) < 2:
        print(('Usage: {} <rosbag> [<outfile>] \n\n'
               'Print contents of rosbag file. If <outfile> is provided, \n'
               'write contents of rosbag file to <outfile> in the legacy \n'
               'lidar binary format LIDAR_CONVERTED_DTYPE.\n'
               'Output files will written with the extension \'{}\'.'
               ).format(__file__, OUTFILE_EXT))
        sys.exit(1)

    outfile = None
    filename = sys.argv[1]

    with rosbag.Bag(filename) as bag:
        if len(sys.argv) >= 3:
            import pickle
            import gzip

            # convert to outfile if provided
            outfile = sys.argv[2]
            record = convert_bag(bag)

            if not outfile.endswith(OUTFILE_EXT):
                outfile += OUTFILE_EXT
            pickle.dump(record, gzip.open(outfile, 'wb'))
        else:
            print_bag(bag)

    sys.exit()

<|endoftext|>


<commit_before># Copyright (c) 2013, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright notice,
#       this list of conditions and the following disclaimer in the documentation
#       and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""readline module."""

import bits
import bits.input
import pager
import redirect
import string
import sys

__all__ = ["init", "get_completer", "set_completer", "parse_and_bind"]

width = []
height = []
line_x = []
line_y = []
buffer_max = 0
history = []
ctrl_o_index = None
completer = None

def init():
    """Initialize the readline module and configure it as the Python readline callback."""
    from _bits import _set_readline_callback
    _set_readline_callback(_readline)

def insert_char(line_buffer, c, pos):
    global buffer_max
    if len(line_buffer) < buffer_max:
        line_buffer = line_buffer[:pos] + c + line_buffer[pos:]
        pos = pos + 1
    return line_buffer, pos

def delete_char(line_buffer, pos):
    if len(line_buffer) > 0:
        line_buffer = line_buffer[:pos] + line_buffer[pos+1:]
    return line_buffer, pos

def delete_char_left(line_buffer, pos):
    if pos == 0:
        return line_buffer, pos
    line_buffer, temp_pos = delete_char(line_buffer, pos - 1)
    return line_buffer, pos - 1

def print_buffer(line_buffer, x, y, term):
    global width, height, line_y
    curr_pos = 0
    while curr_pos < len(line_buffer):
        bits.goto_xy(x, y, term)
        max_width = width[term] - 1 - x
        newline = False
        if len(line_buffer) - curr_pos >= max_width:
            partial_count = max_width
            x = 0
            y = y + 1
            newline = True
        else:
            partial_count = len(line_buffer) - curr_pos
            x = x + partial_count
        bits.puts(line_buffer[curr_pos:curr_pos + partial_count], term)
        if newline:
            bits.puts('\n', term)
            # check for scroll here and adjust start position
            if y == height[term]:
                y = y - 1
                line_y[term] = line_y[term] - 1
        curr_pos = curr_pos + partial_count

def PositionCursor(pos, x, y, term):
    global width
    curr_pos = 0
    while curr_pos < pos:
        max_width = width[term] - 1 - x
        if pos - curr_pos >= max_width:
            partial_count = max_width
            x = 0
            y = y + 1
        else:
            partial_count = pos - curr_pos
            x = x + partial_count
        curr_pos = curr_pos + partial_count
    bits.goto_xy(x, y, term)
    return(x, y)

def get_completer():
    """Get the readline completer function."""

    global completer
    return completer

def set_completer(f):
    """Set the readline completer function."""

    global completer
    completer = f

def parse_and_bind(s):
    """Stub parse_and_bind() function for compatibility with readline.

    This module does not support readline's configuration syntax, so this
    function does nothing."""
    return None

key_hooks = {}
function_keys = set(getattr(bits.input, "KEY_F{}".format(n)) for n in range(1, 12+1))

def add_key_hook(key, func):
    global key_hooks
    assert key in function_keys
    assert key_hooks.get(key) in (None, func)
    key_hooks[key] = func

def _readline(prompt=""):
    global width, height, line_x, line_y, buffer_max, history, ctrl_o_index, completer

    with redirect.nolog():
        with pager.nopager():
            sys.stdout.write(prompt)

            line_buffer = ''
            pos = 0
            prev_len = 0

            term_count = bits.get_term_count()
            width = [0] * term_count
            height = [0] * term_count
            line_x = [0] * term_count
            line_y = [0] * term_count

            for term in range(term_count):
                width[term], height[term] = bits.get_width_height(term)
                line_x[term], line_y[term] = bits.get_xy(term)

            buffer_max = min((width[term] - 2 - line_x[term]) + ((height[term] - 1) * (width[term] - 1)) for term in range(term_count))

            history_index = len(history)
            history_state = dict()
            completer_state = 0

            if ctrl_o_index is not None:
                if ctrl_o_index < len(history):
                    history_index = ctrl_o_index
                    line_buffer = history[history_index]
                    pos = len(line_buffer)
                ctrl_o_index = None

            while True:
                # Update history
                history_state[history_index] = (line_buffer, pos)

                try:
                    # clear any characters after the current line buffer
                    trailing_len = prev_len - len(line_buffer)
                    if trailing_len > 0:
                        for term in range(term_count):
                            trailing_x, trailing_y = PositionCursor(len(line_buffer), line_x[term], line_y[term], term)
                            print_buffer(" " * trailing_len, trailing_x, trailing_y, term)
                    prev_len = len(line_buffer)

                    for term in range(term_count):
                        # print the current line buffer
                        print_buffer(line_buffer, line_x[term], line_y[term], term)
                        # move the cursor to location of pos within the line buffer
                        PositionCursor(pos, line_x[term], line_y[term], term)

                    c = bits.input.get_key()

                    key = bits.input.key
                    def ctrl(k):
                        return key(k, ctrl=True)

                    if c == key('\r') or c == key('\n') or c == ctrl('o'):
                        if line_buffer or (history and history[-1]):
                            history.append(line_buffer)
                        if c == ctrl('o'): # Ctrl-O
                            ctrl_o_index = history_index + 1
                        sys.stdout.write('\n')
                        return line_buffer + '\n'

                    if not (c == key('\t') or c == ctrl('i')):
                        # reset completer state to force restart of the completer
                        completer_state = 0

                    if c == key(bits.input.KEY_HOME) or c == ctrl('a'):
                        # start of line
                        pos = 0
                    elif c == key(bits.input.KEY_LEFT) or c == ctrl('b'):
                        # left
                        if pos != 0:
                            pos -= 1
                    elif c == ctrl('d'):
                        # EOF
                        if len(line_buffer) == 0:
                            return ""
                        if pos < len(line_buffer):
                            line_buffer, pos = delete_char(line_buffer, pos)
                    elif c == key(bits.input.KEY_DELETE):
                        if pos < len(line_buffer):
                            line_buffer, pos = delete_char(line_buffer, pos)
                    elif c == key(bits.input.KEY_END) or c == ctrl('e'):
                        # end of line
                        pos = len(line_buffer)
                    elif c == key(bits.input.KEY_RIGHT) or c == ctrl('f'):
                        # right
                        if pos != len(line_buffer):
                            pos += 1
                    elif c == key('\b') or c == ctrl('h'):
                        # backspace
                        line_buffer, pos = delete_char_left(line_buffer, pos)
                    elif c == key('\t') or c == ctrl('i'):
                        # tab completion
                        if completer is not None:
                            if completer_state != 0:
                                for c in range(len(current_completion)):
                                    line_buffer, pos = delete_char_left(line_buffer, pos)
                            else:
                                cur = pos
                                while pos != 0 and line_buffer[pos-1] != ' ':
                                    pos -= 1
                                saved_str = line_buffer[pos:cur]
                                line_buffer = line_buffer[:pos] + line_buffer[cur:]
                            current_completion = completer(saved_str, completer_state)
                            completer_state += 1
                            if current_completion is not None:
                                for c in current_completion:
                                    line_buffer, pos = insert_char(line_buffer, c, pos)
                            else:
                                for c in saved_str:
                                    line_buffer, pos = insert_char(line_buffer, c, pos)
                                completer_state = 0
                    elif c == ctrl('k'):
                        # delete from current to end of line
                        line_buffer = line_buffer[:pos]
                    elif c == ctrl('l'):
                        # clear screen
                        bits.clear_screen()
                        sys.stdout.write(prompt)
                        for term in range(term_count):
                            line_x[term], line_y[term] = bits.get_xy(term);
                    elif c == key(bits.input.KEY_DOWN) or c == ctrl('n'):
                        # Next line in history
                        if history_index < len(history):
                            history_index += 1
                            if history_index == len(history):
                                line_buffer, pos = history_state.get(history_index)
                            else:
                                line_buffer, pos = history_state.get(history_index, (history[history_index], len(history[history_index])))
                    elif c == key(bits.input.KEY_UP) or c == ctrl('p'):
                        # Previous line in history
                        if history_index > 0:
                            history_index -= 1
                            line_buffer, pos = history_state.get(history_index, (history[history_index], len(history[history_index])))
                    elif c == ctrl('u'):
                        # delete from current to beginning of line
                        line_buffer = line_buffer[pos:]
                        pos = 0
                    elif c == ctrl('w'):
                        # delete previous word
                        cur = pos
                        while pos != 0 and line_buffer[pos-1] == ' ':
                            pos -= 1
                        while pos != 0 and line_buffer[pos-1] != ' ':
                            pos -= 1
                        line_buffer = line_buffer[:pos] + line_buffer[cur:]
                    elif c == ctrl('z') or c == key(bits.input.KEY_ESC):
                        if len(line_buffer) == 0:
                            return ""
                    elif c.key in key_hooks:
                        key_hooks[c.key]()
                    elif not(c.ctrl) and not(c.alt) and isinstance(c.key, basestring) and c.key in string.printable:
                        # printable
                        try:
                            line_buffer, pos = insert_char(line_buffer, c.key.encode('ascii'), pos)
                        except UnicodeError:
                            pass
                    else:
                        pass

                except IOError:
                    pass
<commit_msg>python: Fix line history navigation with Ctrl-O<commit_after># Copyright (c) 2013, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright notice,
#       this list of conditions and the following disclaimer in the documentation
#       and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""readline module."""

import bits
import bits.input
import pager
import redirect
import string
import sys

__all__ = ["init", "get_completer", "set_completer", "parse_and_bind"]

width = []
height = []
line_x = []
line_y = []
buffer_max = 0
history = []
ctrl_o_index = None
completer = None

def init():
    """Initialize the readline module and configure it as the Python readline callback."""
    from _bits import _set_readline_callback
    _set_readline_callback(_readline)

def insert_char(line_buffer, c, pos):
    global buffer_max
    if len(line_buffer) < buffer_max:
        line_buffer = line_buffer[:pos] + c + line_buffer[pos:]
        pos = pos + 1
    return line_buffer, pos

def delete_char(line_buffer, pos):
    if len(line_buffer) > 0:
        line_buffer = line_buffer[:pos] + line_buffer[pos+1:]
    return line_buffer, pos

def delete_char_left(line_buffer, pos):
    if pos == 0:
        return line_buffer, pos
    line_buffer, temp_pos = delete_char(line_buffer, pos - 1)
    return line_buffer, pos - 1

def print_buffer(line_buffer, x, y, term):
    global width, height, line_y
    curr_pos = 0
    while curr_pos < len(line_buffer):
        bits.goto_xy(x, y, term)
        max_width = width[term] - 1 - x
        newline = False
        if len(line_buffer) - curr_pos >= max_width:
            partial_count = max_width
            x = 0
            y = y + 1
            newline = True
        else:
            partial_count = len(line_buffer) - curr_pos
            x = x + partial_count
        bits.puts(line_buffer[curr_pos:curr_pos + partial_count], term)
        if newline:
            bits.puts('\n', term)
            # check for scroll here and adjust start position
            if y == height[term]:
                y = y - 1
                line_y[term] = line_y[term] - 1
        curr_pos = curr_pos + partial_count

def PositionCursor(pos, x, y, term):
    global width
    curr_pos = 0
    while curr_pos < pos:
        max_width = width[term] - 1 - x
        if pos - curr_pos >= max_width:
            partial_count = max_width
            x = 0
            y = y + 1
        else:
            partial_count = pos - curr_pos
            x = x + partial_count
        curr_pos = curr_pos + partial_count
    bits.goto_xy(x, y, term)
    return(x, y)

def get_completer():
    """Get the readline completer function."""

    global completer
    return completer

def set_completer(f):
    """Set the readline completer function."""

    global completer
    completer = f

def parse_and_bind(s):
    """Stub parse_and_bind() function for compatibility with readline.

    This module does not support readline's configuration syntax, so this
    function does nothing."""
    return None

key_hooks = {}
function_keys = set(getattr(bits.input, "KEY_F{}".format(n)) for n in range(1, 12+1))

def add_key_hook(key, func):
    global key_hooks
    assert key in function_keys
    assert key_hooks.get(key) in (None, func)
    key_hooks[key] = func

def _readline(prompt=""):
    global width, height, line_x, line_y, buffer_max, history, ctrl_o_index, completer

    with redirect.nolog():
        with pager.nopager():
            sys.stdout.write(prompt)

            line_buffer = ''
            pos = 0
            prev_len = 0

            term_count = bits.get_term_count()
            width = [0] * term_count
            height = [0] * term_count
            line_x = [0] * term_count
            line_y = [0] * term_count

            for term in range(term_count):
                width[term], height[term] = bits.get_width_height(term)
                line_x[term], line_y[term] = bits.get_xy(term)

            buffer_max = min((width[term] - 2 - line_x[term]) + ((height[term] - 1) * (width[term] - 1)) for term in range(term_count))

            history_index = len(history)
            history_state = dict()
            completer_state = 0

            if ctrl_o_index is not None:
                if ctrl_o_index < len(history):
                    history_index = ctrl_o_index
                    line_buffer = history[history_index]
                    pos = len(line_buffer)
                ctrl_o_index = None

            while True:
                # Update history
                history_state[history_index] = (line_buffer, pos)

                try:
                    # clear any characters after the current line buffer
                    trailing_len = prev_len - len(line_buffer)
                    if trailing_len > 0:
                        for term in range(term_count):
                            trailing_x, trailing_y = PositionCursor(len(line_buffer), line_x[term], line_y[term], term)
                            print_buffer(" " * trailing_len, trailing_x, trailing_y, term)
                    prev_len = len(line_buffer)

                    for term in range(term_count):
                        # print the current line buffer
                        print_buffer(line_buffer, line_x[term], line_y[term], term)
                        # move the cursor to location of pos within the line buffer
                        PositionCursor(pos, line_x[term], line_y[term], term)

                    c = bits.input.get_key()

                    key = bits.input.key
                    def ctrl(k):
                        return key(k, ctrl=True)

                    if c == key('\r') or c == key('\n') or c == ctrl('o'):
                        if line_buffer or (history and history[-1]):
                            history.append(line_buffer)
                        if c == ctrl('o'): # Ctrl-O
                            ctrl_o_index = history_index + 1
                        sys.stdout.write('\n')
                        return line_buffer + '\n'

                    if not (c == key('\t') or c == ctrl('i')):
                        # reset completer state to force restart of the completer
                        completer_state = 0

                    if c == key(bits.input.KEY_HOME) or c == ctrl('a'):
                        # start of line
                        pos = 0
                    elif c == key(bits.input.KEY_LEFT) or c == ctrl('b'):
                        # left
                        if pos != 0:
                            pos -= 1
                    elif c == ctrl('d'):
                        # EOF
                        if len(line_buffer) == 0:
                            return ""
                        if pos < len(line_buffer):
                            line_buffer, pos = delete_char(line_buffer, pos)
                    elif c == key(bits.input.KEY_DELETE):
                        if pos < len(line_buffer):
                            line_buffer, pos = delete_char(line_buffer, pos)
                    elif c == key(bits.input.KEY_END) or c == ctrl('e'):
                        # end of line
                        pos = len(line_buffer)
                    elif c == key(bits.input.KEY_RIGHT) or c == ctrl('f'):
                        # right
                        if pos != len(line_buffer):
                            pos += 1
                    elif c == key('\b') or c == ctrl('h'):
                        # backspace
                        line_buffer, pos = delete_char_left(line_buffer, pos)
                    elif c == key('\t') or c == ctrl('i'):
                        # tab completion
                        if completer is not None:
                            if completer_state != 0:
                                for c in range(len(current_completion)):
                                    line_buffer, pos = delete_char_left(line_buffer, pos)
                            else:
                                cur = pos
                                while pos != 0 and line_buffer[pos-1] != ' ':
                                    pos -= 1
                                saved_str = line_buffer[pos:cur]
                                line_buffer = line_buffer[:pos] + line_buffer[cur:]
                            current_completion = completer(saved_str, completer_state)
                            completer_state += 1
                            if current_completion is not None:
                                for c in current_completion:
                                    line_buffer, pos = insert_char(line_buffer, c, pos)
                            else:
                                for c in saved_str:
                                    line_buffer, pos = insert_char(line_buffer, c, pos)
                                completer_state = 0
                    elif c == ctrl('k'):
                        # delete from current to end of line
                        line_buffer = line_buffer[:pos]
                    elif c == ctrl('l'):
                        # clear screen
                        bits.clear_screen()
                        sys.stdout.write(prompt)
                        for term in range(term_count):
                            line_x[term], line_y[term] = bits.get_xy(term);
                    elif c == key(bits.input.KEY_DOWN) or c == ctrl('n'):
                        # Next line in history
                        if history_index < len(history):
                            history_index += 1
                            if history_index == len(history):
                                line_buffer, pos = history_state.get(history_index, ('', 0))
                            else:
                                line_buffer, pos = history_state.get(history_index, (history[history_index], len(history[history_index])))
                    elif c == key(bits.input.KEY_UP) or c == ctrl('p'):
                        # Previous line in history
                        if history_index > 0:
                            history_index -= 1
                            line_buffer, pos = history_state.get(history_index, (history[history_index], len(history[history_index])))
                    elif c == ctrl('u'):
                        # delete from current to beginning of line
                        line_buffer = line_buffer[pos:]
                        pos = 0
                    elif c == ctrl('w'):
                        # delete previous word
                        cur = pos
                        while pos != 0 and line_buffer[pos-1] == ' ':
                            pos -= 1
                        while pos != 0 and line_buffer[pos-1] != ' ':
                            pos -= 1
                        line_buffer = line_buffer[:pos] + line_buffer[cur:]
                    elif c == ctrl('z') or c == key(bits.input.KEY_ESC):
                        if len(line_buffer) == 0:
                            return ""
                    elif c.key in key_hooks:
                        key_hooks[c.key]()
                    elif not(c.ctrl) and not(c.alt) and isinstance(c.key, basestring) and c.key in string.printable:
                        # printable
                        try:
                            line_buffer, pos = insert_char(line_buffer, c.key.encode('ascii'), pos)
                        except UnicodeError:
                            pass
                    else:
                        pass

                except IOError:
                    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
from distutils.core import setup
import os

OKGREEN = '\033[92m'
FAIL = '\033[91m'
WARNING = '\033[93m'
ENDC = '\033[0m'

def get_hptt_root():
    """Get the root hptt folder - prefer environment variable if is set, but
    otherwise default to the parent directory of the python module - the
    shared library will be checked for in any case.
    """
    from os.path import dirname, realpath, isfile, join
    hptt_default = dirname(dirname(realpath(__file__)))
    HPTT_ROOT = os.environ.get('HPTT_ROOT', hptt_default)
    if not isfile(join(HPTT_ROOT, 'lib', 'libhptt.so')):
        raise OSError("Could not find $HPTT_ROOT/lib/libhptt.so - please make "
                      "HPTT_ROOT points to a directory where the shared "
                      "library has been built.")
    return HPTT_ROOT

def write_config():
    """Write a config file with the default shared library location to be
    installed by setup.py.
    """
    from configparser import ConfigParser
    config = ConfigParser()
    config['lib'] = {'HPTT_ROOT': get_hptt_root()}
    with open(os.path.join("hptt", "hptt.cfg"), "w") as configfile:
        config.write(configfile)

write_config()

setup(
    name="hptt",
    version="1.0.0",
    description="High-Performance Tensor Transposition",
    author="Paul Springer",
    author_email="springer@aices.rwth-aachen.de",
    packages=["hptt"],
    package_data={'hptt': ['hptt.cfg']},
)

print("")
output = "# "+ FAIL + "IMPORTANT"+ENDC+": execute 'export HPTT_ROOT=%s/../' #"%(os.path.dirname(os.path.realpath(__file__)))
print('#'*(len(output)-2*len(FAIL)+1))
print(output)
print('#'*(len(output)-2*len(FAIL)+1))
print("")
<commit_msg>specify numpy and psutil in requires<commit_after>#!/usr/bin/env python
from distutils.core import setup
import os

OKGREEN = '\033[92m'
FAIL = '\033[91m'
WARNING = '\033[93m'
ENDC = '\033[0m'

def get_hptt_root():
    """Get the root hptt folder - prefer environment variable if is set, but
    otherwise default to the parent directory of the python module - the
    shared library will be checked for in any case.
    """
    from os.path import dirname, realpath, isfile, join
    hptt_default = dirname(dirname(realpath(__file__)))
    HPTT_ROOT = os.environ.get('HPTT_ROOT', hptt_default)
    if not isfile(join(HPTT_ROOT, 'lib', 'libhptt.so')):
        raise OSError("Could not find $HPTT_ROOT/lib/libhptt.so - please make "
                      "HPTT_ROOT points to a directory where the shared "
                      "library has been built.")
    return HPTT_ROOT

def write_config():
    """Write a config file with the default shared library location to be
    installed by setup.py.
    """
    from configparser import ConfigParser
    config = ConfigParser()
    config['lib'] = {'HPTT_ROOT': get_hptt_root()}
    with open(os.path.join("hptt", "hptt.cfg"), "w") as configfile:
        config.write(configfile)

write_config()

setup(
    name="hptt",
    version="1.0.0",
    description="High-Performance Tensor Transposition",
    author="Paul Springer",
    author_email="springer@aices.rwth-aachen.de",
    packages=["hptt"],
    package_data={'hptt': ['hptt.cfg']},
    install_requires=[
        'numpy',
        'psutil',
    ]
)

print("")
output = "# "+ FAIL + "IMPORTANT"+ENDC+": execute 'export HPTT_ROOT=%s/../' #"%(os.path.dirname(os.path.realpath(__file__)))
print('#'*(len(output)-2*len(FAIL)+1))
print(output)
print('#'*(len(output)-2*len(FAIL)+1))
print("")
<|endoftext|>