<commit_before>import unittest
import sys
import imp
import os
import datetime
import re
import logging
import copy
from xml.dom.minidom import parse, parseString
from xml_patent import XMLPatent
from optparse import OptionParser
from sql_patent import SQLPatent
from types import *

# Data structures/variables used in testing
tables = ["assignee", "citation", "class", "inventor",
          "patent", "patdesc", "lawyer", "sciref", "usreldoc"]
debug = False
xml_files = []
parsed_xml = []
max_years = "2012"
max_months = "12"
max_days = "31"
first_patent = "17900731"

dir = os.path.dirname(__file__)
folder = os.path.join(dir, 'unittest/')
log_file = os.path.join(dir, 'unittest/log/unit-test-log.log')
xml_files = [x for x in os.listdir(folder)
             if re.match(r".*?patent.*?xml", x) != None]

# Logging setup
logging.basicConfig(filename=log_file, level=logging.DEBUG)

# TODO:
# Update xml presence tests, make it more robust.

"""
Each individual test in unit-testing should be able to run individually.
"""

class TestXMLPatent(unittest.TestCase):

    def setUp(self):
        # Basic sanity check
        self.assertTrue(xml_files)

    def test_patent_XML_construction(self):
        # High-level test, testing legacy code construction,
        # if doesn't construct obviously won't pass other tests, fail-fast mentality
        if debug:
            print "\n     Testing Well-formedness and Construction\n"
        logging.info("Testing Construction of %d Patents!" % (len(xml_files)))
        patent_count = 0
        for i, xml in enumerate(xml_files):
            if debug:
                print " - Testing Patent: %s" % (xml)
            try:
                file_to_open = open(folder + xml, 'U')
            except Exception as fileError:
                logging.error("Error opening patent %d, filename: %s"
                             % (i+1, xml))
            try:
                xml_patent = XMLPatent(file_to_open)
            except Exception as exPatError:
                logging.error("Construction Error at patent %d, filename %s"
                             % (i+1, xml))
            # Storing tuple (original XML file, parsed XML) for finer block testing
            parsed_xml.append(xml_patent)

    def test_patent_SQL_tblBuild_asg(self):
        parsed_xml = []
        for xml in xml_files:
            parsed_xml.append(XMLPatent(open(folder + xml, 'U')))
        list_of_tables = []
        testSQL = SQLPatent()
        new_table = testSQL.tblBuild(parsed_xml, "assignee")
        # print "new table is...", new_table, len(new_table)
        for xml in parsed_xml:
            asg_list = xml.asg_list[0]
            # print "asg list is...", asg_list
            for table_entry in new_table:
                self.assertTrue(len(table_entry) == 9)
                self.assertTrue(table_entry[0] == xml.patent)
                self.assertTrue(table_entry[1] == asg_list[2])
                self.assertTrue(table_entry[2] == asg_list[1])
                self.assertTrue(table_entry[3] == asg_list[4])
                self.assertTrue(table_entry[4] == asg_list[5])
                self.assertTrue(table_entry[5] == asg_list[6])
                self.assertTrue(table_entry[6] == asg_list[7])
                self.assertTrue(table_entry[7] == asg_list[8])

    def test_patent_SQL_tblBuild_cit(self):
        parsed_xml = []
        for xml in xml_files:
            parsed_xml.append(XMLPatent(open(folder + xml, 'U')))
        list_of_tables = []
        testSQL = SQLPatent()
        new_table = testSQL.tblBuild(parsed_xml, "citation")
        for xml in parsed_xml:
            cit_list = [y for y in xml.cit_list if y[1]!=""][0]
            # print "cit_list is...", cit_list
            # print "new_table is...:", new_table
            for table_entry in new_table:
                self.assertTrue(len(table_entry) == 8)
                self.assertTrue(table_entry[0] == xml.patent)
                # print "table entry[2]:", table_entry[2]
                # print "cit_list[5]", cit_list[5]
                # self.assertTrue(table_entry[2] == cit_list[5])
                # self.assertTrue(table_entry[3] == cit_list[4])
                # self.assertTrue(table_entry[4] == cit_list[1])
                # self.assertTrue(table_entry[5] == cit_list[2])
                # self.assertTrue(table_entry[6] == cit_list[0])

    def test_patent_SQL_tblBuild_class(self):
        parsed_xml = []
        for xml in xml_files:
            parsed_xml.append(XMLPatent(open(folder + xml, 'U')))
        list_of_tables = []
        testSQL = SQLPatent()
        for xml in parsed_xml:
            print xml.classes

    def tearDown(self):
        #anything needed to be torn down should be added here, pass for now
        pass

if __name__ == '__main__':

    parser = OptionParser()
    parser.add_option("-d", "--debugging", dest="debugging", action="store_true")
    (options, args) = parser.parse_args()

    """
        Complications passing in command-line arguments along with unit-testing,
        Need to delete sys.argv from index one onewards or else flag keeps getting
        interpreted, solution found
        http://stackoverflow.com/questions/1029891/python-unittest-is-there-a-way-to-pass-command-line-options-to-the-app
    """

    if options.debugging:
        debug = True
    del sys.argv[1:]
    if debug:
        print "\n     Starting Unit Testing for XMLPatent()"
    open(log_file, 'w')
    unittest.main()
<commit_msg>fixed minor iterating bug<commit_after>import unittest
import sys
import imp
import os
import datetime
import re
import logging
import copy
from xml.dom.minidom import parse, parseString
from xml_patent import XMLPatent
from optparse import OptionParser
from sql_patent import SQLPatent
from types import *

# Data structures/variables used in testing
tables = ["assignee", "citation", "class", "inventor",
          "patent", "patdesc", "lawyer", "sciref", "usreldoc"]
debug = False
xml_files = []
parsed_xml = []
max_years = "2012"
max_months = "12"
max_days = "31"
first_patent = "17900731"

dir = os.path.dirname(__file__)
folder = os.path.join(dir, 'unittest/')
log_file = os.path.join(dir, 'unittest/log/unit-test-log.log')
xml_files = [x for x in os.listdir(folder)
             if re.match(r".*?patent.*?xml", x) != None]

# Logging setup
logging.basicConfig(filename=log_file, level=logging.DEBUG)

# TODO:
# Update xml presence tests, make it more robust.

"""
Each individual test in unit-testing should be able to run individually.
"""

class TestXMLPatent(unittest.TestCase):

    def setUp(self):
        # Basic sanity check
        self.assertTrue(xml_files)

    def test_patent_XML_construction(self):
        # High-level test, testing legacy code construction,
        # if doesn't construct obviously won't pass other tests, fail-fast mentality
        if debug:
            print "\n     Testing Well-formedness and Construction\n"
        logging.info("Testing Construction of %d Patents!" % (len(xml_files)))
        patent_count = 0
        for i, xml in enumerate(xml_files):
            if debug:
                print " - Testing Patent: %s" % (xml)
            try:
                file_to_open = open(folder + xml, 'U')
            except Exception as fileError:
                logging.error("Error opening patent %d, filename: %s"
                             % (i+1, xml))
            try:
                xml_patent = XMLPatent(file_to_open)
            except Exception as exPatError:
                logging.error("Construction Error at patent %d, filename %s"
                             % (i+1, xml))
            # Storing tuple (original XML file, parsed XML) for finer block testing
            parsed_xml.append(xml_patent)

    def test_patent_SQL_tblBuild_asg(self):
        parsed_xml = []
        for xml in xml_files:
            parsed_xml.append(XMLPatent(open(folder + xml, 'U')))
        list_of_tables = []
        testSQL = SQLPatent()
        new_table = testSQL.tblBuild(parsed_xml, "assignee")
        # print "new table is...", new_table, len(new_table)
        for table_entry in new_table:
            for xml in parsed_xml:
                asg_list = xml.asg_list[0]
                self.assertTrue(len(table_entry) == 9)
                self.assertTrue(table_entry[0] == xml.patent)
                self.assertTrue(table_entry[1] == asg_list[2])
                self.assertTrue(table_entry[2] == asg_list[1])
                self.assertTrue(table_entry[3] == asg_list[4])
                self.assertTrue(table_entry[4] == asg_list[5])
                self.assertTrue(table_entry[5] == asg_list[6])
                self.assertTrue(table_entry[6] == asg_list[7])
                self.assertTrue(table_entry[7] == asg_list[8])

    def test_patent_SQL_tblBuild_asg2(self):
        parsed_xml = []
        for xml in xml_files:
            parsed_xml.append(XMLPatent(open(folder + xml, 'U')))
        list_of_tables = []
        testSQL = SQLPatent()
        new_table = testSQL.tblBuild(parsed_xml, "assignee")
        # print "new table is...", new_table, len(new_table)
        for i, table_entry in enumerate(new_table):
            for xml in parsed_xml:
                asg_list = xml.asg_list[i]
                self.assertTrue(len(table_entry) == 9)
                self.assertTrue(table_entry[0] == xml.patent)
                self.assertTrue(table_entry[1] == asg_list[2])
                self.assertTrue(table_entry[2] == asg_list[1])
                self.assertTrue(table_entry[3] == asg_list[4])
                self.assertTrue(table_entry[4] == asg_list[5])
                self.assertTrue(table_entry[5] == asg_list[6])
                self.assertTrue(table_entry[6] == asg_list[7])
                self.assertTrue(table_entry[7] == asg_list[8])

    def test_patent_SQL_tblBuild_cit(self):
        parsed_xml = []
        for xml in xml_files:
            parsed_xml.append(XMLPatent(open(folder + xml, 'U')))
        list_of_tables = []
        testSQL = SQLPatent()
        new_table = testSQL.tblBuild(parsed_xml, "citation")
        for i, table_entry in enumerate(new_table):
            for xml in parsed_xml:
                cit_list = xml.cit_list[i]
                self.assertTrue(len(table_entry) == 8)
                self.assertTrue(table_entry[0] == xml.patent)
                # print "table entry[2]:", table_entry[2]
                # print "cit_list[5]", cit_list[5]
                self.assertTrue(table_entry[2] == cit_list[5])
                self.assertTrue(table_entry[3] == cit_list[4])
                self.assertTrue(table_entry[4] == cit_list[1])
                self.assertTrue(table_entry[5] == cit_list[2])
                self.assertTrue(table_entry[6] == cit_list[0])

    def test_patent_SQL_tblBuild_class(self):
        parsed_xml = []
        for xml in xml_files:
            parsed_xml.append(XMLPatent(open(folder + xml, 'U')))
        list_of_tables = []
        testSQL = SQLPatent()
        new_table = testSQL.tblBuild(parsed_xml, "class")
        for i, table_entry in enumerate(new_table):
            for xml in parsed_xml:
                class_list = xml.classes[i]
                self.assertTrue(len(table_entry) == 4)
                self.assertTrue(table_entry[0] == xml.patent)
                self.assertTrue(table_entry[1] == (i==0)*1)
                self.assertTrue(table_entry[2] == class_list[0])
                self.assertTrue(table_entry[3] == class_list[1])

    def tearDown(self):
        #anything needed to be torn down should be added here, pass for now
        pass

if __name__ == '__main__':

    parser = OptionParser()
    parser.add_option("-d", "--debugging", dest="debugging", action="store_true")
    (options, args) = parser.parse_args()

    """
        Complications passing in command-line arguments along with unit-testing,
        Need to delete sys.argv from index one onewards or else flag keeps getting
        interpreted, solution found
        http://stackoverflow.com/questions/1029891/python-unittest-is-there-a-way-to-pass-command-line-options-to-the-app
    """

    if options.debugging:
        debug = True
    del sys.argv[1:]
    if debug:
        print "\n     Starting Unit Testing for XMLPatent()"
    open(log_file, 'w')
    unittest.main()
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2022 The Fiddle-Config Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Functions to output representations of `fdl.Buildable`s."""

import dataclasses
import inspect
from typing import Any, Iterator, List, Optional, Sequence, Tuple, Type, Union

from fiddle import config
from fiddle import history
from fiddle import placeholders
from fiddle.codegen import formatting_utilities
import tree

@dataclasses.dataclass(frozen=True)
class _ParamName:
  """Wrapper for a string to differentiate Buildable param names from dict keys."""
  __slots__ = ('name',)
  name: str

@dataclasses.dataclass(frozen=True)
class _UnsetValue:
  __slots__ = ('parameter',)
  parameter: inspect.Parameter

  def __repr__(self) -> str:
    if self.parameter.default is self.parameter.empty:
      return '<[unset]>'
    else:
      return f'<[unset; default: {self.parameter.default!r}]>'

_Leaf = Any
_Path = Sequence[Union[int, str, _ParamName]]

def _path_to_str(path: _Path) -> str:
  """Converts a path to a string representation."""
  output = ''
  for item in path:
    if isinstance(item, _ParamName):
      if output:
        output += '.' + item.name
      else:
        # First item.
        output = item.name
    else:
      output += f'[{item!r}]'
  return output

def _has_nested_builder(children: Sequence[Tuple[_Path, _Leaf]]) -> bool:
  return any(
      isinstance(child, config.Buildable) for unused_path, child in children)

def _format_value(value: Any, *, raw_value_repr: bool) -> str:
  if raw_value_repr:
    return repr(value)
  else:
    if isinstance(value, str):
      return repr(value)
    else:
      return formatting_utilities.pretty_print(value)

@dataclasses.dataclass(frozen=True)
class _PlaceholderWrapper:
  """Customizes representation for placeholders in flattened output."""
  __slots__ = ('wrapped',)
  wrapped: placeholders.Placeholder

  def __repr__(self):
    if self.wrapped.value is placeholders.NO_VALUE:
      value_repr = '<[unset]>'
    else:
      value_repr = repr(self.wrapped.value)
    return (f'fdl.Placeholder({self.wrapped.key.name!r}, '
            f'value={value_repr})')

def as_str_flattened(cfg: config.Buildable,
                     *,
                     include_types: bool = True,
                     raw_value_repr: bool = False) -> str:
  """Returns a string of cfg's paths and values, one pair per line.

  Some automated tools (e.g. grep, sort, diff, ...) handle data line-by-line.
  This output format contains the full path to a value and a string
  representation of said value on a single line.

  Args:
    cfg: A buildable to generate a string representation for.
    include_types: If true, include type annotations (where available) in the
      string representation of each leaf value.
    raw_value_repr: If true, use `repr` on values, otherwise, a custom
      pretty-printed format is used.
  Returns: a string representation of `cfg`.
  """

  def flatten_children(
      value: Any, annotation: Optional[Type[Any]],
      path: _Path) -> Iterator[Tuple[_Path, Optional[Type[Any]], _Leaf]]:
    flattened_children = tree.flatten_with_path(value)
    if _has_nested_builder(flattened_children):
      for child_path, leaf in flattened_children:
        if isinstance(leaf, placeholders.Placeholder):
          yield path + child_path, None, _PlaceholderWrapper(leaf)
        elif isinstance(leaf, config.Buildable):
          for subpath, subannotation, actual_leaf in recursive_flatten(leaf):
            yield path + child_path + subpath, subannotation, actual_leaf
        else:
          yield path + child_path, None, leaf
    else:
      yield path, annotation, value

  def recursive_flatten(
      buildable: config.Buildable
  ) -> Iterator[Tuple[_Path, Optional[Type[Any]], _Leaf]]:
    signature = buildable.__signature__
    kwarg_param = None
    for param_name, param in signature.parameters.items():
      if param.kind == param.VAR_KEYWORD:
        kwarg_param = param
        continue
      annotation = None if param.annotation is param.empty else param.annotation
      path = (_ParamName(param_name),)
      if param_name not in buildable.__arguments__:
        yield path, annotation, _UnsetValue(param)
      else:
        yield from flatten_children(buildable.__arguments__[param_name],
                                    annotation, path)
    kwarg_annotation = None
    if kwarg_param and kwarg_param.annotation is not kwarg_param.empty:
      kwarg_annotation = kwarg_param.annotation
    not_yielded_argument_names = (
        buildable.__arguments__.keys() - signature.parameters.keys())
    for name in sorted(not_yielded_argument_names):
      path = (_ParamName(name),)
      yield from flatten_children(buildable.__arguments__[name],
                                  kwarg_annotation, path)

  def format_line(line: Tuple[_Path, Optional[Type[Any]], _Leaf]):
    type_annotation = ''
    if include_types and line[1] is not None:
      try:
        type_annotation = f': {line[1].__qualname__}'
      except AttributeError:
        # Certain types, such as Union, do not have a __qualname__ attribute.
        type_annotation = f': {line[1]}'
    value = _format_value(line[2], raw_value_repr=raw_value_repr)
    return f'{_path_to_str(line[0])}{type_annotation} = {value}'

  return '\n'.join(map(format_line, recursive_flatten(cfg)))

def history_per_leaf_parameter(cfg: config.Buildable,
                               *,
                               raw_value_repr: bool = False) -> str:
  """Returns a string representing the history of cfg's leaf params.

  Because Buildable's are designed to be mutated, tracking down when and where
  a particular parameter's value is changed can be difficult. This function
  returns a string representation of each parameter and (in
  reverse-chronological order) its current and past values.

  This representation elides the "DAG"-construction aspects of constructing the
  `cfg`, and instead only prints the history of the "outer-most" parameters
  (ones that don't contain a reference to another Buildable).

  Args:
    cfg: A buildable to generate a history for.
    raw_value_repr: If true, use `repr` when string-ifying values, otherwise use
      a customized pretty-printing routine.

  Returns:
    A string representation of `cfg`'s history, organized by param name.
  """
  return '\n'.join(
      _make_per_leaf_histories_recursive(cfg, raw_value_repr=raw_value_repr))

def _make_per_leaf_histories_recursive(
    cfg: config.Buildable,
    raw_value_repr: bool,
    path: Optional[_Path] = None,
) -> Iterator[str]:
  """Recursively traverses `cfg` and generates per-param history summaries."""
  for name, param_history in cfg.__argument_history__.items():
    if path:
      param_path = tuple(path) + (_ParamName(name),)
    else:
      param_path = (_ParamName(name),)
    children = tree.flatten_with_path(getattr(cfg, name))
    if _has_nested_builder(children):
      for child_path, child_leaf in children:
        full_child_path = param_path + child_path
        if isinstance(child_leaf, config.Buildable):
          yield from _make_per_leaf_histories_recursive(child_leaf,
                                                        raw_value_repr,
                                                        full_child_path)
        else:
          yield f'{_path_to_str(full_child_path)} = {child_leaf}'
    else:
      yield _make_per_leaf_history_text(param_path, param_history,
                                        raw_value_repr)

  unset_fields = sorted(
      set(cfg.__signature__.parameters.keys()) -
      set(cfg.__argument_history__.keys()))
  for name in unset_fields:
    if path:
      param_path = tuple(path) + (_ParamName(name),)
    else:
      param_path = (_ParamName(name),)
    yield f'{_path_to_str(param_path)} = <[unset]>'

def _make_per_leaf_history_text(path: _Path,
                                param_history: List[history.HistoryEntry],
                                raw_value_repr: bool) -> str:
  """Returns a string representing a parameter's history.

  Args:
    path: The path to the parameter.
    param_history: The parameter's history.
    raw_value_repr: If True, use `repr` unmodified, otherwise, use a custom
      pretty-printing routine.

  Returns:
    A string representation of the parameter's history that can be displayed to
    the user.
  """
  assert param_history, 'param_history should never be empty.'

  def make_previous_text(entry: history.HistoryEntry) -> str:
    value = _format_value(entry.value, raw_value_repr=raw_value_repr)
    return f'  - previously: {value} @ {entry.location}'

  if len(param_history) > 1:
    past = '\n'.join(map(make_previous_text, reversed(param_history[:-1])))
    past = '\n' + past  # prefix with a newline.
  else:
    past = ''
  current_value = _format_value(
      param_history[-1].value, raw_value_repr=raw_value_repr)
  current = f'{current_value} @ {param_history[-1].location}'
  return f'{_path_to_str(path)} = {current}{past}'
<commit_msg>Pretty print default values<commit_after># coding=utf-8
# Copyright 2022 The Fiddle-Config Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Functions to output representations of `fdl.Buildable`s."""

import dataclasses
import inspect
from typing import Any, Iterator, List, Optional, Sequence, Tuple, Type, Union

from fiddle import config
from fiddle import history
from fiddle import placeholders
from fiddle.codegen import formatting_utilities
import tree

@dataclasses.dataclass(frozen=True)
class _ParamName:
  """Wrapper for a string to differentiate Buildable param names from dict keys."""
  __slots__ = ('name',)
  name: str

@dataclasses.dataclass(frozen=True)
class _UnsetValue:
  """A wrapper class indicating an unset value."""
  __slots__ = ('parameter',)
  parameter: inspect.Parameter

  def __repr__(self) -> str:
    if self.parameter.default is self.parameter.empty:
      return '<[unset]>'
    else:
      default_value = _format_value(
          self.parameter.default, raw_value_repr=False)
      return f'<[unset; default: {default_value}]>'

_Leaf = Any
_Path = Sequence[Union[int, str, _ParamName]]

def _path_to_str(path: _Path) -> str:
  """Converts a path to a string representation."""
  output = ''
  for item in path:
    if isinstance(item, _ParamName):
      if output:
        output += '.' + item.name
      else:
        # First item.
        output = item.name
    else:
      output += f'[{item!r}]'
  return output

def _has_nested_builder(children: Sequence[Tuple[_Path, _Leaf]]) -> bool:
  return any(
      isinstance(child, config.Buildable) for unused_path, child in children)

def _format_value(value: Any, *, raw_value_repr: bool) -> str:
  if raw_value_repr:
    return repr(value)
  else:
    if isinstance(value, str):
      return repr(value)
    else:
      return formatting_utilities.pretty_print(value)

@dataclasses.dataclass(frozen=True)
class _PlaceholderWrapper:
  """Customizes representation for placeholders in flattened output."""
  __slots__ = ('wrapped',)
  wrapped: placeholders.Placeholder

  def __repr__(self):
    if self.wrapped.value is placeholders.NO_VALUE:
      value_repr = '<[unset]>'
    else:
      value_repr = repr(self.wrapped.value)
    return (f'fdl.Placeholder({self.wrapped.key.name!r}, '
            f'value={value_repr})')

def as_str_flattened(cfg: config.Buildable,
                     *,
                     include_types: bool = True,
                     raw_value_repr: bool = False) -> str:
  """Returns a string of cfg's paths and values, one pair per line.

  Some automated tools (e.g. grep, sort, diff, ...) handle data line-by-line.
  This output format contains the full path to a value and a string
  representation of said value on a single line.

  Args:
    cfg: A buildable to generate a string representation for.
    include_types: If true, include type annotations (where available) in the
      string representation of each leaf value.
    raw_value_repr: If true, use `repr` on values, otherwise, a custom
      pretty-printed format is used.
  Returns: a string representation of `cfg`.
  """

  def flatten_children(
      value: Any, annotation: Optional[Type[Any]],
      path: _Path) -> Iterator[Tuple[_Path, Optional[Type[Any]], _Leaf]]:
    flattened_children = tree.flatten_with_path(value)
    if _has_nested_builder(flattened_children):
      for child_path, leaf in flattened_children:
        if isinstance(leaf, placeholders.Placeholder):
          yield path + child_path, None, _PlaceholderWrapper(leaf)
        elif isinstance(leaf, config.Buildable):
          for subpath, subannotation, actual_leaf in recursive_flatten(leaf):
            yield path + child_path + subpath, subannotation, actual_leaf
        else:
          yield path + child_path, None, leaf
    else:
      yield path, annotation, value

  def recursive_flatten(
      buildable: config.Buildable
  ) -> Iterator[Tuple[_Path, Optional[Type[Any]], _Leaf]]:
    signature = buildable.__signature__
    kwarg_param = None
    for param_name, param in signature.parameters.items():
      if param.kind == param.VAR_KEYWORD:
        kwarg_param = param
        continue
      annotation = None if param.annotation is param.empty else param.annotation
      path = (_ParamName(param_name),)
      if param_name not in buildable.__arguments__:
        yield path, annotation, _UnsetValue(param)
      else:
        yield from flatten_children(buildable.__arguments__[param_name],
                                    annotation, path)
    kwarg_annotation = None
    if kwarg_param and kwarg_param.annotation is not kwarg_param.empty:
      kwarg_annotation = kwarg_param.annotation
    not_yielded_argument_names = (
        buildable.__arguments__.keys() - signature.parameters.keys())
    for name in sorted(not_yielded_argument_names):
      path = (_ParamName(name),)
      yield from flatten_children(buildable.__arguments__[name],
                                  kwarg_annotation, path)

  def format_line(line: Tuple[_Path, Optional[Type[Any]], _Leaf]):
    type_annotation = ''
    if include_types and line[1] is not None:
      try:
        type_annotation = f': {line[1].__qualname__}'
      except AttributeError:
        # Certain types, such as Union, do not have a __qualname__ attribute.
        type_annotation = f': {line[1]}'
    value = _format_value(line[2], raw_value_repr=raw_value_repr)
    return f'{_path_to_str(line[0])}{type_annotation} = {value}'

  return '\n'.join(map(format_line, recursive_flatten(cfg)))

def history_per_leaf_parameter(cfg: config.Buildable,
                               *,
                               raw_value_repr: bool = False) -> str:
  """Returns a string representing the history of cfg's leaf params.

  Because Buildable's are designed to be mutated, tracking down when and where
  a particular parameter's value is changed can be difficult. This function
  returns a string representation of each parameter and (in
  reverse-chronological order) its current and past values.

  This representation elides the "DAG"-construction aspects of constructing the
  `cfg`, and instead only prints the history of the "outer-most" parameters
  (ones that don't contain a reference to another Buildable).

  Args:
    cfg: A buildable to generate a history for.
    raw_value_repr: If true, use `repr` when string-ifying values, otherwise use
      a customized pretty-printing routine.

  Returns:
    A string representation of `cfg`'s history, organized by param name.
  """
  return '\n'.join(
      _make_per_leaf_histories_recursive(cfg, raw_value_repr=raw_value_repr))

def _make_per_leaf_histories_recursive(
    cfg: config.Buildable,
    raw_value_repr: bool,
    path: Optional[_Path] = None,
) -> Iterator[str]:
  """Recursively traverses `cfg` and generates per-param history summaries."""
  for name, param_history in cfg.__argument_history__.items():
    if path:
      param_path = tuple(path) + (_ParamName(name),)
    else:
      param_path = (_ParamName(name),)
    children = tree.flatten_with_path(getattr(cfg, name))
    if _has_nested_builder(children):
      for child_path, child_leaf in children:
        full_child_path = param_path + child_path
        if isinstance(child_leaf, config.Buildable):
          yield from _make_per_leaf_histories_recursive(child_leaf,
                                                        raw_value_repr,
                                                        full_child_path)
        else:
          yield f'{_path_to_str(full_child_path)} = {child_leaf}'
    else:
      yield _make_per_leaf_history_text(param_path, param_history,
                                        raw_value_repr)

  unset_fields = sorted(
      set(cfg.__signature__.parameters.keys()) -
      set(cfg.__argument_history__.keys()))
  for name in unset_fields:
    if path:
      param_path = tuple(path) + (_ParamName(name),)
    else:
      param_path = (_ParamName(name),)
    yield f'{_path_to_str(param_path)} = <[unset]>'

def _make_per_leaf_history_text(path: _Path,
                                param_history: List[history.HistoryEntry],
                                raw_value_repr: bool) -> str:
  """Returns a string representing a parameter's history.

  Args:
    path: The path to the parameter.
    param_history: The parameter's history.
    raw_value_repr: If True, use `repr` unmodified, otherwise, use a custom
      pretty-printing routine.

  Returns:
    A string representation of the parameter's history that can be displayed to
    the user.
  """
  assert param_history, 'param_history should never be empty.'

  def make_previous_text(entry: history.HistoryEntry) -> str:
    value = _format_value(entry.value, raw_value_repr=raw_value_repr)
    return f'  - previously: {value} @ {entry.location}'

  if len(param_history) > 1:
    past = '\n'.join(map(make_previous_text, reversed(param_history[:-1])))
    past = '\n' + past  # prefix with a newline.
  else:
    past = ''
  current_value = _format_value(
      param_history[-1].value, raw_value_repr=raw_value_repr)
  current = f'{current_value} @ {param_history[-1].location}'
  return f'{_path_to_str(path)} = {current}{past}'
<|endoftext|>


<commit_before><commit_msg>Update simplenote.py to 1.0.4<commit_after><|endoftext|>


<commit_before>__author__ = 'McGuiT1'
import sys
import time
import pygame
import os
from pygame.locals import *
import random

BLOCKSIZE=15
x = 100
y = 20
os.environ['SDL_VIDEO_WINDOW_POS'] = "%d,%d" % (x,y)
pygame.mixer.pre_init(44100, -16, 2, 2048) # setup mixer to avoid sound lag
pygame.init()
soundFile = "snare.wav"
pygame.mixer.init()
sound = pygame.mixer.Sound(soundFile)
#window:
WINWIDTH = 1200
WINHEIGHT = 700
winSurf = pygame.display.set_mode((WINWIDTH,WINHEIGHT),0,32)
pygame.display.set_caption('Animation!!!!!!')

DOWNLEFT = 1
DOWNRIGHT = 3
UPLEFT = 7
UPRIGHT = 9
NONE=0
UP = 2
DOWN = 8
MOVESPEED = 2

BLACK = (0,0,0)
WHITE = (255,255,255)
RED = (255,0,0)
GREEN = (0,255,0)
BLUE = (0,0,255)
ORANGE = (255,128,20)
PINK = (255,10,255)
#Startup screen:
startup = True
settings = False
basicFont = pygame.font.SysFont(None, 40)
text = basicFont.render('Start: a', True, WHITE, BLUE)
text2 = basicFont.render('Settings: c', True, WHITE, BLUE)
def display_settings():

    # settings has speed, blocksize, fade
    settingsList = []

    settingsList.append("Start Speed: s")
    settingsList.append("Block Size: b")
    settingsList.append("Fade: f")
    settingsFont = pygame.font.SysFont(None,30)
    y_offset = 60
    for str in settingsList:
        tt = settingsFont.render(str,True, WHITE, BLUE)
        ttRect = tt.get_rect()
        ttRect.centerx = winSurf.get_rect().centerx
        ttRect.centery = winSurf.get_rect().centery + y_offset
        y_offset = y_offset + 60

        winSurf.blit(tt,ttRect)

textRect = text.get_rect()
textRect.centerx = winSurf.get_rect().centerx
textRect.centery = winSurf.get_rect().centery - 200

text2Rect = text2.get_rect()
text2Rect.centerx = winSurf.get_rect().centerx
text2Rect.centery = winSurf.get_rect().centery - 100

winSurf.blit(text,textRect)
winSurf.blit(text2,text2Rect)
pygame.display.update()
while startup:
    for event in pygame.event.get():

        if event.type == pygame.locals.QUIT:
            pygame.quite()
            sys.exit()

        if event.type == pygame.locals.KEYDOWN:
            if event.key == ord('a'):
                startup = False
            if event.key == ord('c'):
                display_settings()
                pygame.display.update()
            if event.key == pygame.locals.K_ESCAPE:
                pygame.quit()
                sys.exit()

p1color = PINK
p2color = ORANGE
def crash(x,y):
    particles = []
    # send 10 - 20 particles at random directions
    for b in range (0,10):
        particle = {'rect':pygame.Rect(x,y,2,2), 'color':RED, 'dir':UP }
        particles.append(particle)
    return particles

def isPointInsideRect(x,y,rect):
    if (x >= rect.left) and (x <= rect.right) and (y > rect.top) and (y < rect.bottom):
        return True

def collision(player,sb):
    for block in sb:
        for a,b in [(player, block), (block, player)]:
            if ((isPointInsideRect (a['rect'].left, a['rect'].top, b['rect'])) or
                (isPointInsideRect (a['rect'].left, a['rect'].bottom, b['rect'])) or
                (isPointInsideRect(a['rect'].right, a['rect'].top, b['rect'])) or (isPointInsideRect(a['rect'].right, a['rect'].bottom, b['rect']))):
                return True

def moveBlock(b):

    if b['dir'] == DOWNLEFT:
        b['rect'].left -= MOVESPEED
        b['rect'].top += MOVESPEED
    if b['dir'] == UP:
        b['rect'].top -= MOVESPEED
    if b['dir'] == DOWNRIGHT:
        b['rect'].left += MOVESPEED
        b['rect'].top += MOVESPEED
    if b['dir'] == UPRIGHT:
        b['rect'].left += MOVESPEED
        b['rect'].top -= MOVESPEED
    if b['dir'] == UPLEFT:
        b['rect'].left -= MOVESPEED
        b['rect'].top -= MOVESPEED

    if b['rect'].bottom < 0:
        #past the top
        b['rect'].top = WINHEIGHT

    if b['rect'].bottom > WINHEIGHT:
        #past the bottom
        if b['dir'] == DOWNLEFT:
            b['dir'] = UPLEFT
        if b['dir'] == DOWNRIGHT:
            b['dir'] = UPRIGHT

    if b['rect'].left < 0:
        b['rect'].left = WINWIDTH -BLOCKSIZE

    if b['rect'].right > WINWIDTH:
        b['rect'].left =0

def restart(player1points, player2points):
    time.sleep(0.3)
    player1 = {'rect':pygame.Rect(300,80,BLOCKSIZE,BLOCKSIZE), 'color':GREEN, 'bcolor':PINK, 'dir':UP, 'points':player1points}
    player2 = {'rect':pygame.Rect(WINWIDTH/2 + 100,100,BLOCKSIZE,BLOCKSIZE), 'color':BLUE, 'bcolor':ORANGE, 'dir':UP, 'points':player2points}
    return player1, player2

def doStaticBlocks():
    static_blocks = []
    blocks_to_skip = random.randint(5,10)
    start = random.randint(0,WINHEIGHT - blocks_to_skip * BLOCKSIZE)
    end = start + blocks_to_skip * BLOCKSIZE

    for x in range(0,WINHEIGHT,BLOCKSIZE):
        # skip blocks in a row at random:
        if x < start or x > end:
            static_blocks.append({'rect':pygame.Rect(WINWIDTH/2,x,BLOCKSIZE,BLOCKSIZE),'color':WHITE, 'dir':NONE})
    return static_blocks

def makeStaticBlock(static_blocks,player):
    h = 0
    v = 0
    playerColor = player['bcolor']
    # put a static block down and advance player past it so it doesn't register as a collision
    # if player direction is up, put it BLOCKSIZE to the rear  # if upleft, put it down and right  if upright, put it down and right
    if player['dir'] == UP:
        h = 0
        v = BLOCKSIZE
    if player['dir'] == UPLEFT:
        h = BLOCKSIZE
        v = BLOCKSIZE
    if player['dir'] == UPRIGHT:
        h = -BLOCKSIZE
        v = BLOCKSIZE

    newBlock = {'rect':pygame.Rect(player['rect'].left + h,player['rect'].top + v,BLOCKSIZE,BLOCKSIZE),'color':playerColor, 'dir':NONE}
    static_blocks.append(newBlock)
    return static_blocks

player1 = {'rect':pygame.Rect(300,80,BLOCKSIZE,BLOCKSIZE), 'color':RED,'bcolor':PINK, 'dir':UP, 'points':0}
player2 = {'rect':pygame.Rect(WINWIDTH/2 + 100,100,BLOCKSIZE,BLOCKSIZE), 'color':BLUE, 'bcolor':ORANGE, 'dir':UP, 'points':0}
static_blocks = doStaticBlocks()
p1addblock = False
p2addblock = False

#gameloop
while True:
    for event in pygame.event.get():

        if event.type == pygame.locals.QUIT:
            print ("player1: " + str(player1['points']))
            print ("player2: " + str(player2['points']))
            pygame.quit()
            sys.exit()

        if event.type == pygame.locals.KEYDOWN:
            if event.key == pygame.locals.K_ESCAPE:
                pygame.quit()
                sys.exit()
            if event.key == pygame.locals.K_LEFT:
                player2['dir'] = UPLEFT
            if event.key == pygame.locals.K_RIGHT:
                player2['dir'] = UPRIGHT
            if event.key == pygame.locals.K_UP:
                player2['dir'] = UP
            if event.key == pygame.locals.K_DOWN:
                p2addblock = True

            if event.key == ord('a'):
                player1['dir'] = UPLEFT
            if event.key == ord('d'):
                player1['dir'] = UPRIGHT
            if event.key == ord('w'):
                player1['dir'] = UP
            if event.key == ord('s'):
                p1addblock = True
        if event.type == pygame.locals.KEYUP:

            if event.key == pygame.locals.K_DOWN:
                p2addblock = False
            if event.key == ord('s'):
                p1addblock = False

    winSurf.fill(BLACK)

    if p2addblock:
        makeStaticBlock(static_blocks,player2)
#        newBlock = {'rect':pygame.Rect(player2['rect'].left-20,player2['rect'].top-20,20,20),'color':PINK, 'dir':NONE}
#        static_blocks.append(newBlock)

    if p1addblock:
 #       newBlock = {'rect':pygame.Rect(player1['rect'].left-20,player1['rect'].top -20,20,20),'color':ORANGE, 'dir':NONE}
 #       static_blocks.append(newBlock)
        makeStaticBlock(static_blocks,player1)
    moveBlock(player1)
    moveBlock(player2)

    hasCollision = False
    if(collision(player1, static_blocks)):
        player2['points'] += 1
        hasCollision = True
    if(collision(player2,static_blocks)):
        player1['points'] += 1
        hasCollision = True
    if hasCollision:
        sound.play()
        MOVESPEED = 3
        static_blocks = doStaticBlocks()
        player1,player2 = restart(player1['points'], player2['points'])
    if MOVESPEED < 10:
        MOVESPEED = MOVESPEED + 0.1

    pygame.draw.rect(winSurf,player1['color'],player1['rect'])
    pygame.draw.rect(winSurf,player2['color'],player2['rect'])
    for sb in static_blocks:
        pygame.draw.rect(winSurf,sb['color'],sb['rect'])
    pygame.display.update()
    time.sleep(0.02)

<commit_msg>work on menu<commit_after>__author__ = 'McGuiT1'
import sys
import time
import pygame
import os
from pygame.locals import *
import random

BLOCKSIZE=15
x = 100
y = 20
os.environ['SDL_VIDEO_WINDOW_POS'] = "%d,%d" % (x,y)
pygame.mixer.pre_init(44100, -16, 2, 2048) # setup mixer to avoid sound lag
pygame.init()
soundFile = "snare.wav"
pygame.mixer.init()
sound = pygame.mixer.Sound(soundFile)
#window:
WINWIDTH = 1200
WINHEIGHT = 700
winSurf = pygame.display.set_mode((WINWIDTH,WINHEIGHT),0,32)
pygame.display.set_caption('Animation!!!!!!')

DOWNLEFT = 1
DOWNRIGHT = 3
UPLEFT = 7
UPRIGHT = 9
NONE=0
UP = 2
DOWN = 8
MOVESPEED = 2

BLACK = (0,0,0)
WHITE = (255,255,255)
RED = (255,0,0)
GREEN = (0,255,0)
BLUE = (0,0,255)
ORANGE = (255,128,20)
PINK = (255,10,255)
#Startup screen:
startup = True
settings = False

def menu_render(textList,startPos,sepDist,fontSize=40):
    basicFont = pygame.font.SysFont(None, fontSize)
    for str in textList:
        tt = basicFont.render(str,True, WHITE,BLUE)
        ttRect = tt.get_rect()
        ttRect.centerx = winSurf.get_rect().centerx
        ttRect.centery = winSurf.get_rect().centery + startPos
        startPos = startPos + sepDist
        winSurf.blit(tt,ttRect)

menuList = ["Start: a", "Settings: c"]

menu_render(menuList,-200,100)

def display_settings():
    # settings has speed, blocksize, fade
    settingsList = ["Start Speed: s","Block Size: b","Fade: f" ]
    menu_render(settingsList,60,60,30)

pygame.display.update()
while startup:
    for event in pygame.event.get():
        if event.type == pygame.locals.QUIT:
            pygame.quite()
            sys.exit()

        if event.type == pygame.locals.KEYDOWN:
            if event.key == ord('a'):
                startup = False
            if event.key == ord('c'):
                display_settings()
                pygame.display.update()
            if event.key == pygame.locals.K_ESCAPE:
                pygame.quit()
                sys.exit()

p1color = PINK
p2color = ORANGE
def crash(x,y):
    particles = []
    # send 10 - 20 particles at random directions
    for b in range (0,10):
        particle = {'rect':pygame.Rect(x,y,2,2), 'color':RED, 'dir':UP }
        particles.append(particle)
    return particles

def isPointInsideRect(x,y,rect):
    if (x >= rect.left) and (x <= rect.right) and (y > rect.top) and (y < rect.bottom):
        return True

def collision(player,sb):
    for block in sb:
        for a,b in [(player, block), (block, player)]:
            if ((isPointInsideRect (a['rect'].left, a['rect'].top, b['rect'])) or
                (isPointInsideRect (a['rect'].left, a['rect'].bottom, b['rect'])) or
                (isPointInsideRect(a['rect'].right, a['rect'].top, b['rect'])) or (isPointInsideRect(a['rect'].right, a['rect'].bottom, b['rect']))):
                return True

def moveBlock(b):

    if b['dir'] == DOWNLEFT:
        b['rect'].left -= MOVESPEED
        b['rect'].top += MOVESPEED
    if b['dir'] == UP:
        b['rect'].top -= MOVESPEED
    if b['dir'] == DOWNRIGHT:
        b['rect'].left += MOVESPEED
        b['rect'].top += MOVESPEED
    if b['dir'] == UPRIGHT:
        b['rect'].left += MOVESPEED
        b['rect'].top -= MOVESPEED
    if b['dir'] == UPLEFT:
        b['rect'].left -= MOVESPEED
        b['rect'].top -= MOVESPEED

    if b['rect'].bottom < 0:
        #past the top
        b['rect'].top = WINHEIGHT

    if b['rect'].bottom > WINHEIGHT:
        #past the bottom
        if b['dir'] == DOWNLEFT:
            b['dir'] = UPLEFT
        if b['dir'] == DOWNRIGHT:
            b['dir'] = UPRIGHT

    if b['rect'].left < 0:
        b['rect'].left = WINWIDTH -BLOCKSIZE

    if b['rect'].right > WINWIDTH:
        b['rect'].left =0

def restart(player1points, player2points):
    time.sleep(0.3)
    player1 = {'rect':pygame.Rect(300,80,BLOCKSIZE,BLOCKSIZE), 'color':GREEN, 'bcolor':PINK, 'dir':UP, 'points':player1points}
    player2 = {'rect':pygame.Rect(WINWIDTH/2 + 100,100,BLOCKSIZE,BLOCKSIZE), 'color':BLUE, 'bcolor':ORANGE, 'dir':UP, 'points':player2points}
    return player1, player2

def doStaticBlocks():
    static_blocks = []
    blocks_to_skip = random.randint(5,10)
    start = random.randint(0,WINHEIGHT - blocks_to_skip * BLOCKSIZE)
    end = start + blocks_to_skip * BLOCKSIZE

    for x in range(0,WINHEIGHT,BLOCKSIZE):
        # skip blocks in a row at random:
        if x < start or x > end:
            static_blocks.append({'rect':pygame.Rect(WINWIDTH/2,x,BLOCKSIZE,BLOCKSIZE),'color':WHITE, 'dir':NONE})
    return static_blocks

def makeStaticBlock(static_blocks,player):
    h = 0
    v = 0
    playerColor = player['bcolor']
    # put a static block down and advance player past it so it doesn't register as a collision
    # if player direction is up, put it BLOCKSIZE to the rear  # if upleft, put it down and right  if upright, put it down and right
    if player['dir'] == UP:
        h = 0
        v = BLOCKSIZE
    if player['dir'] == UPLEFT:
        h = BLOCKSIZE
        v = BLOCKSIZE
    if player['dir'] == UPRIGHT:
        h = -BLOCKSIZE
        v = BLOCKSIZE

    newBlock = {'rect':pygame.Rect(player['rect'].left + h,player['rect'].top + v,BLOCKSIZE,BLOCKSIZE),'color':playerColor, 'dir':NONE}
    static_blocks.append(newBlock)
    return static_blocks

player1 = {'rect':pygame.Rect(300,80,BLOCKSIZE,BLOCKSIZE), 'color':RED,'bcolor':PINK, 'dir':UP, 'points':0}
player2 = {'rect':pygame.Rect(WINWIDTH/2 + 100,100,BLOCKSIZE,BLOCKSIZE), 'color':BLUE, 'bcolor':ORANGE, 'dir':UP, 'points':0}
static_blocks = doStaticBlocks()
p1addblock = False
p2addblock = False

#gameloop
while True:
    for event in pygame.event.get():

        if event.type == pygame.locals.QUIT:
            print ("player1: " + str(player1['points']))
            print ("player2: " + str(player2['points']))
            pygame.quit()
            sys.exit()

        if event.type == pygame.locals.KEYDOWN:
            if event.key == pygame.locals.K_ESCAPE:
                pygame.quit()
                sys.exit()
            if event.key == pygame.locals.K_LEFT:
                player2['dir'] = UPLEFT
            if event.key == pygame.locals.K_RIGHT:
                player2['dir'] = UPRIGHT
            if event.key == pygame.locals.K_UP:
                player2['dir'] = UP
            if event.key == pygame.locals.K_DOWN:
                p2addblock = True

            if event.key == ord('a'):
                player1['dir'] = UPLEFT
            if event.key == ord('d'):
                player1['dir'] = UPRIGHT
            if event.key == ord('w'):
                player1['dir'] = UP
            if event.key == ord('s'):
                p1addblock = True
        if event.type == pygame.locals.KEYUP:

            if event.key == pygame.locals.K_DOWN:
                p2addblock = False
            if event.key == ord('s'):
                p1addblock = False

    winSurf.fill(BLACK)

    if p2addblock:
        makeStaticBlock(static_blocks,player2)
#        newBlock = {'rect':pygame.Rect(player2['rect'].left-20,player2['rect'].top-20,20,20),'color':PINK, 'dir':NONE}
#        static_blocks.append(newBlock)

    if p1addblock:
 #       newBlock = {'rect':pygame.Rect(player1['rect'].left-20,player1['rect'].top -20,20,20),'color':ORANGE, 'dir':NONE}
 #       static_blocks.append(newBlock)
        makeStaticBlock(static_blocks,player1)
    moveBlock(player1)
    moveBlock(player2)

    hasCollision = False
    if(collision(player1, static_blocks)):
        player2['points'] += 1
        hasCollision = True
    if(collision(player2,static_blocks)):
        player1['points'] += 1
        hasCollision = True
    if hasCollision:
        sound.play()
        MOVESPEED = 3
        static_blocks = doStaticBlocks()
        player1,player2 = restart(player1['points'], player2['points'])
    if MOVESPEED < 10:
        MOVESPEED = MOVESPEED + 0.1

    pygame.draw.rect(winSurf,player1['color'],player1['rect'])
    pygame.draw.rect(winSurf,player2['color'],player2['rect'])
    for sb in static_blocks:
        pygame.draw.rect(winSurf,sb['color'],sb['rect'])
    pygame.display.update()
    time.sleep(0.02)

<|endoftext|>


<commit_before>import pandas as pd
import pkg_resources

data_path = 'https://github.com/autonomio/datasets/raw/master/autonomio-datasets/'

def load_data(name,mode):

    if mode == 'default':

        if name == 'election_in_twitter':
            out = pd.read_msgpack(data_path+'election_in_twitter')
        if name == 'sites_category_and_vec':
            out = pd.read_msgpack(data_path+'sites_category_and_vec')
        if name == 'programmatic_ad_fraud':
            out = pd.read_msgpack(data_path+'programmatic_ad_fraud')
        if name == 'parties_and_employment':
            out = pd.read_msgpack(data_path+'parties_and_employment')
        if name == 'tweet_sentiment':
            out = pd.read_msgpack(data_path+'tweet_sentiment')
        if name == 'random_tweets':
            out = pd.read_msgpack(data_path+'random_tweets')

        return out

    elif mode == 'file':

        if name.endswith('.csv'):

            try:
                out = pd.read_csv(name)
            except:
                out = pd.read_csv(name,error_bad_lines=False)

        if name.endswith('.msg'):
            out = pd.read_msgpack(name)

        if name.endswith('.json'):
            out = pd.read_json(name)

        if name.endswith('.xls'):
            out = pd.read_excel(name)

        else:

            out = ''
            print "MAYBE UNSUPPORTED? only csv, json, xls, and msgpack"

        return out<commit_msg>Fixed load data file<commit_after>import pandas as pd
import pkg_resources

data_path = 'https://github.com/autonomio/datasets/raw/master/autonomio-datasets/'

def load_data(name,mode):

    if mode == 'default':

        if name == 'election_in_twitter':
            out = pd.read_msgpack(data_path+'election_in_twitter')
        if name == 'sites_category_and_vec':
            out = pd.read_msgpack(data_path+'sites_category_and_vec')
        if name == 'programmatic_ad_fraud':
            out = pd.read_msgpack(data_path+'programmatic_ad_fraud')
        if name == 'parties_and_employment':
            out = pd.read_msgpack(data_path+'parties_and_employment')
        if name == 'tweet_sentiment':
            out = pd.read_msgpack(data_path+'tweet_sentiment')
        if name == 'random_tweets':
            out = pd.read_msgpack(data_path+'random_tweets')

        return out

    elif mode == 'file':

        if name.endswith('.csv'):

            try:
                out = pd.read_csv(name)
            except:
                out = pd.read_csv(name,error_bad_lines=False)

        elif name.endswith('.msgpack'):
            out = pd.read_msgpack(name)

        elif name.endswith('.json'):
            out = pd.read_json(name)

        else:

            out = ''
            print "MAYBE UNSUPPORTED? only csv, json, and msgpack"

        return out<|endoftext|>


<commit_before>import re
import unicodedata
from socketio import socketio_manage
from socketio.namespace import BaseNamespace
from socketio.mixins import RoomsMixin, BroadcastMixin
from werkzeug.exceptions import NotFound
from gevent import monkey
from os import urandom

from flask import Flask, Response, request, render_template, url_for, redirect
from flask.ext.sqlalchemy import SQLAlchemy

monkey.patch_all()

app = Flask(__name__)
app.debug = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/chat.db'
db = SQLAlchemy(app)

# models
class ChatRoom(db.Model):
    __tablename__ = 'chatrooms'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(20), nullable=False)
    slug = db.Column(db.String(50))
    users = db.relationship('ChatUser', backref='chatroom', lazy='dynamic')

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return url_for('room', slug=self.slug)

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)
        db.session.add(self)
        db.session.commit()

class ChatUser(db.Model):
    __tablename__ = 'chatusers'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(20), nullable=False)
    session = db.Column(db.String(20), nullable=False)
    chatroom_id = db.Column(db.Integer, db.ForeignKey('chatrooms.id'))

    def __unicode__(self):
        return self.name

# utils
def slugify(value):
    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
    value = unicode(re.sub('[^\w\s-]', '', value).strip().lower())
    return re.sub('[-\s]+', '-', value)

def get_object_or_404(klass, **query):
    instance = klass.query.filter_by(**query).first()
    if not instance:
        raise NotFound()
    return instance

def get_or_create(klass, **kwargs):
    try:
        return get_object_or_404(klass, **kwargs), False
    except NotFound:
        instance = klass(**kwargs)
        instance.save()
        return instance, True

def init_db():
    db.create_all(app=app)

# views
@app.route('/')
def rooms():
    """
    Homepage - lists all rooms.
    """
    context = {"rooms": ChatRoom.query.all()}
    return render_template('rooms.html', **context)

@app.route('/<path:slug>')
def room(slug):
    """
    Show a room.
    """
    context = {"room": get_object_or_404(ChatRoom, slug=slug)}
    return render_template('room.html', **context)

@app.route('/create_room')
def create_room():
    room_name = urandom(20)
    room, created = get_or_create(ChatRoom, name=room_name)
    return "Response: OK!\n" + room_name, 200

@app.route('/create', methods=['POST'])
def create():
    """
    Handles post from the "Add room" form on the homepage, and
    redirects to the new room.
    """
    name = request.form.get("name")
    if name:
        room, created = get_or_create(ChatRoom, name=name)
        return redirect(url_for('room', slug=room.slug))
    return redirect(url_for('rooms'))

@app.route('/license_consultee')
def get_consultee_license():
    license_consultee_file = open("license_consultee.txt", "r")
    return license_consultee_file.read(), 200

class ChatNamespace(BaseNamespace, RoomsMixin, BroadcastMixin):
    nicknames = []

    def initialize(self):
        self.logger = app.logger
        self.log("Socketio session started")

    def log(self, message):
        self.logger.info("[{0}] {1}".format(self.socket.sessid, message))

    def on_join(self, room):
        self.room = room
        self.join(room)
        return True

    def on_nickname(self, nickname):
        self.log('Nickname: {0}'.format(nickname))
        self.nicknames.append(nickname)
        self.session['nickname'] = nickname
        self.emit_to_room(self.room, 'msg_to_room',
            nickname, 'connected')
        return True, nickname

    def recv_disconnect(self):
        # Remove nickname from the list.
        self.log('Disconnected')
        nickname = self.session['nickname']
        self.nicknames.remove(nickname)
        self.emit_to_room(self.room, 'msg_to_room',
            nickname, 'disconnected')
        self.disconnect(silent=True)
        return True

    def on_user_message(self, msg):
        self.log('User message: {0}'.format(msg))
        self.emit_to_room(self.room, 'msg_to_room',
            self.session['nickname'], msg)
        return True

@app.route('/socket.io/<path:remaining>')
def socketio(remaining):
    try:
        socketio_manage(request.environ, {'/chat': ChatNamespace}, request)
    except:
        app.logger.error("Exception while handling socketio connection",
                         exc_info=True)
    return Response()

if __name__ == '__main__':
    app.run()
<commit_msg>Fixed name generation for rooms.<commit_after>import re
import unicodedata
from socketio import socketio_manage
from socketio.namespace import BaseNamespace
from socketio.mixins import RoomsMixin, BroadcastMixin
from werkzeug.exceptions import NotFound
from gevent import monkey
from os import urandom

from flask import Flask, Response, request, render_template, url_for, redirect
from flask.ext.sqlalchemy import SQLAlchemy

monkey.patch_all()

app = Flask(__name__)
app.debug = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/chat.db'
db = SQLAlchemy(app)

# models
class ChatRoom(db.Model):
    __tablename__ = 'chatrooms'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(20), nullable=False)
    slug = db.Column(db.String(50))
    users = db.relationship('ChatUser', backref='chatroom', lazy='dynamic')

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return url_for('room', slug=self.slug)

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)
        db.session.add(self)
        db.session.commit()

class ChatUser(db.Model):
    __tablename__ = 'chatusers'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(20), nullable=False)
    session = db.Column(db.String(20), nullable=False)
    chatroom_id = db.Column(db.Integer, db.ForeignKey('chatrooms.id'))

    def __unicode__(self):
        return self.name

# utils
def slugify(value):
    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
    value = unicode(re.sub('[^\w\s-]', '', value).strip().lower())
    return re.sub('[-\s]+', '-', value)

def get_object_or_404(klass, **query):
    instance = klass.query.filter_by(**query).first()
    if not instance:
        raise NotFound()
    return instance

def get_or_create(klass, **kwargs):
    try:
        return get_object_or_404(klass, **kwargs), False
    except NotFound:
        instance = klass(**kwargs)
        instance.save()
        return instance, True

def init_db():
    db.create_all(app=app)

def base64_url_encode(input):
 return input.replace('+', '-').replace('/', '_').replace('=', ',')

# views
@app.route('/')
def rooms():
    """
    Homepage - lists all rooms.
    """
    context = {"rooms": ChatRoom.query.all()}
    return render_template('rooms.html', **context)

@app.route('/<path:slug>')
def room(slug):
    """
    Show a room.
    """
    context = {"room": get_object_or_404(ChatRoom, slug=slug)}
    return render_template('room.html', **context)

@app.route('/create_room')
def create_room():
    room_name = base64_url_encode(urandom(20).encode('base64'))
    room, created = get_or_create(ChatRoom, name=room_name)
    return "Response: OK!\n" + room_name, 200

@app.route('/create', methods=['POST'])
def create():
    """
    Handles post from the "Add room" form on the homepage, and
    redirects to the new room.
    """
    name = request.form.get("name")
    if name:
        room, created = get_or_create(ChatRoom, name=name)
        return redirect(url_for('room', slug=room.slug))
    return redirect(url_for('rooms'))

@app.route('/license_consultee')
def get_consultee_license():
    license_consultee_file = open("license_consultee.txt", "r")
    return license_consultee_file.read(), 200

class ChatNamespace(BaseNamespace, RoomsMixin, BroadcastMixin):
    nicknames = []

    def initialize(self):
        self.logger = app.logger
        self.log("Socketio session started")

    def log(self, message):
        self.logger.info("[{0}] {1}".format(self.socket.sessid, message))

    def on_join(self, room):
        self.room = room
        self.join(room)
        return True

    def on_nickname(self, nickname):
        self.log('Nickname: {0}'.format(nickname))
        self.nicknames.append(nickname)
        self.session['nickname'] = nickname
        self.emit_to_room(self.room, 'msg_to_room',
            nickname, 'connected')
        return True, nickname

    def recv_disconnect(self):
        # Remove nickname from the list.
        self.log('Disconnected')
        nickname = self.session['nickname']
        self.nicknames.remove(nickname)
        self.emit_to_room(self.room, 'msg_to_room',
            nickname, 'disconnected')
        self.disconnect(silent=True)
        return True

    def on_user_message(self, msg):
        self.log('User message: {0}'.format(msg))
        self.emit_to_room(self.room, 'msg_to_room',
            self.session['nickname'], msg)
        return True

@app.route('/socket.io/<path:remaining>')
def socketio(remaining):
    try:
        socketio_manage(request.environ, {'/chat': ChatNamespace}, request)
    except:
        app.logger.error("Exception while handling socketio connection",
                         exc_info=True)
    return Response()

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys

sys.path.append("..")
import jauxiliar as jaux
import josecommon as jcommon

CONFIG_HELP = """
José Configuration:
 * `botblock`, if True, blocks all bot messages, use `j!botblock`
 * `language`, string representing the server's language, use `j!language`
"""

class JoseLanguage(jaux.Auxiliar):
    def __init__(self, _client):
        jaux.Auxiliar.__init__(self, _client)
        self.langlist = (
            'pt', 'en'
        )

    async def savedb(self):
        await jcommon.save_configdb()

    async def ext_load(self):
        await self.dbapi.initializedb()
        status = await jcommon.load_configdb()
        return status

    async def ext_unload(self):
        status = await jcommon.save_configdb()
        return status

    async def c_reloadcdb(self, message, args, cxt):
        await self.savedb()
        await jcommon.load_configdb()
        await cxt.say(":speech_left: configdb reloaded")

    async def c_confighelp(self, message, args, cxt):
        await cxt.say(CONFIG_HELP)

    async def c_botblock(self, message, args, cxt):
        '''`j!botblock` - toggles bot block'''
        if message.server is None:
            await cxt.say("Why are you here?")
            return

        sid = message.server.id

        botblock = jcommon.configdb_get(sid, 'botblock')
        if botblock is None:
            self.logger.warning("Botblock is None")

        done = await jcommon.configdb_set(sid, 'botblock', not botblock)
        if not done:
            await cxt.say("Error when changing `botblock` for this server.")
            return

        # sanity check
        n_botblock = jcommon.configdb_get(sid, 'botblock')
        if n_botblock is None:
            self.logger.warning("Botblock is None... again")

        if n_botblock == (not botblock):
            await cxt.say("Botblock from %s to %s", (botblock, not botblock))
        else:
            await cxt.say("No changes to botblock")

    async def c_language(self, message, args, cxt):
        '''`j!language lang` - sets language for a server(use `!listlang` for available languages)'''
        if message.server is None:
            await cxt.say("Language support is not available for DMs")
            return

        if len(args) < 2:
            await cxt.say(self.c_language.__doc__)
            return

        language = args[1]

        if language not in self.langlist:
            await cxt.say("%s: Language not found", (language,))
            return

        await jcommon.langdb_set(message.server.id, language)
        await cxt.say(":speech_left: Set language to %s", (language,))
        await self.savedb()

    async def c_listlang(self, message, args, cxt):
        '''`j!listlang` - lists all available languages'''
        if message.server is None:
            await cxt.say("Language support is not available for DMs")
            return

        llist = self.codeblock("", " ".join(self.langlist))
        serverlang = await jcommon.langdb_get(message.server.id)
        await cxt.say("This server's language: `%s`\nAvailable languages: %s", \
            (serverlang, llist))
<commit_msg>jlang: botblock: await<commit_after>#!/usr/bin/env python3

import sys

sys.path.append("..")
import jauxiliar as jaux
import josecommon as jcommon

CONFIG_HELP = """
José Configuration:
 * `botblock`, if True, blocks all bot messages, use `j!botblock`
 * `language`, string representing the server's language, use `j!language`
"""

class JoseLanguage(jaux.Auxiliar):
    def __init__(self, _client):
        jaux.Auxiliar.__init__(self, _client)
        self.langlist = (
            'pt', 'en'
        )

    async def savedb(self):
        await jcommon.save_configdb()

    async def ext_load(self):
        await self.dbapi.initializedb()
        status = await jcommon.load_configdb()
        return status

    async def ext_unload(self):
        status = await jcommon.save_configdb()
        return status

    async def c_reloadcdb(self, message, args, cxt):
        await self.savedb()
        await jcommon.load_configdb()
        await cxt.say(":speech_left: configdb reloaded")

    async def c_confighelp(self, message, args, cxt):
        await cxt.say(CONFIG_HELP)

    async def c_botblock(self, message, args, cxt):
        '''`j!botblock` - toggles bot block'''
        if message.server is None:
            await cxt.say("Why are you here?")
            return

        sid = message.server.id

        botblock = await jcommon.configdb_get(sid, 'botblock')
        if botblock is None:
            self.logger.warning("Botblock is None")

        done = await jcommon.configdb_set(sid, 'botblock', not botblock)
        if not done:
            await cxt.say("Error when changing `botblock` for this server.")
            return

        # sanity check
        n_botblock = await jcommon.configdb_get(sid, 'botblock')
        if n_botblock is None:
            self.logger.warning("Botblock is None... again")

        if n_botblock == (not botblock):
            await cxt.say("Botblock from %s to %s", (botblock, not botblock))
        else:
            await cxt.say("No changes to botblock")

    async def c_language(self, message, args, cxt):
        '''`j!language lang` - sets language for a server(use `!listlang` for available languages)'''
        if message.server is None:
            await cxt.say("Language support is not available for DMs")
            return

        if len(args) < 2:
            await cxt.say(self.c_language.__doc__)
            return

        language = args[1]

        if language not in self.langlist:
            await cxt.say("%s: Language not found", (language,))
            return

        await jcommon.langdb_set(message.server.id, language)
        await cxt.say(":speech_left: Set language to %s", (language,))
        await self.savedb()

    async def c_listlang(self, message, args, cxt):
        '''`j!listlang` - lists all available languages'''
        if message.server is None:
            await cxt.say("Language support is not available for DMs")
            return

        llist = self.codeblock("", " ".join(self.langlist))
        serverlang = await jcommon.langdb_get(message.server.id)
        await cxt.say("This server's language: `%s`\nAvailable languages: %s", \
            (serverlang, llist))
<|endoftext|>


<commit_before>"""GCM Router"""
import gcmclient
import json

from twisted.internet.threads import deferToThread
from twisted.logger import Logger

from autopush.router.interface import RouterException, RouterResponse
from autopush.senderids import SenderIDs

class GCMRouter(object):
    """GCM Router Implementation"""
    log = Logger()
    gcm = None
    dryRun = 0
    collapseKey = "simplepush"

    def __init__(self, ap_settings, router_conf):
        """Create a new GCM router and connect to GCM"""
        self.config = router_conf
        self.min_ttl = router_conf.get("ttl", 60)
        self.dryRun = router_conf.get("dryrun", False)
        self.collapseKey = router_conf.get("collapseKey", "simplepush")
        self.senderIDs = router_conf.get("senderIDs")
        self.metrics = ap_settings.metrics
        self._base_tags = []
        if not self.senderIDs:
            self.senderIDs = SenderIDs(router_conf)
        try:
            senderID = self.senderIDs.choose_ID()
            self.gcm = gcmclient.GCM(senderID.get("auth"))
        except:
            raise IOError("GCM Bridge not initiated in main")
        self.log.debug("Starting GCM router...")

    def check_token(self, token):
        if token not in self.senderIDs.senderIDs():
            return (False, self.senderIDs.choose_ID().get('senderID'))
        return (True, token)

    def amend_msg(self, msg, data=None):
        if data is not None:
            msg["senderid"] = data.get('creds', {}).get('senderID')
        return msg

    def register(self, uaid, router_data, router_token=None, *kwargs):
        """Validate that the GCM Instance Token is in the ``router_data``"""
        if "token" not in router_data:
            raise self._error("connect info missing GCM Instance 'token'",
                              status=401)
        # Assign a senderid
        router_data["creds"] = self.senderIDs.get_ID(router_token)
        return router_data

    def route_notification(self, notification, uaid_data):
        """Start the GCM notification routing, returns a deferred"""
        router_data = uaid_data["router_data"]
        # Kick the entire notification routing off to a thread
        return deferToThread(self._route, notification, router_data)

    def _route(self, notification, router_data):
        """Blocking GCM call to route the notification"""
        data = {"chid": notification.channel_id}
        # Payload data is optional. The endpoint handler validates that the
        # correct encryption headers are included with the data.
        if notification.data:
            mdata = self.config.get('max_data', 4096)
            if len(notification.data) > mdata:
                raise self._error("This message is intended for a " +
                                  "constrained device and is limited " +
                                  "to 3070 bytes. Converted buffer too " +
                                  "long by %d bytes" %
                                  (len(notification.data) - mdata),
                                  413, errno=104)

            data['body'] = notification.data
            data['con'] = notification.headers['content-encoding']
            data['enc'] = notification.headers['encryption']

            if 'crypto-key' in notification.headers:
                data['cryptokey'] = notification.headers['crypto-key']
            elif 'encryption-key' in notification.headers:
                data['enckey'] = notification.headers['encryption-key']

        # registration_ids are the GCM instance tokens (specified during
        # registration.
        router_ttl = notification.ttl or 0
        payload = gcmclient.JSONMessage(
            registration_ids=[router_data.get("token")],
            collapse_key=self.collapseKey,
            time_to_live=max(self.min_ttl, router_ttl),
            dry_run=self.dryRun or ("dryrun" in router_data),
            data=data,
        )
        creds = router_data.get("creds", {"senderID": "missing id"})
        try:
            self.gcm.api_key = creds["auth"]
            result = self.gcm.send(payload)
        except KeyError:
            raise self._error("Server error, missing bridge credentials " +
                              "for %s" % creds.get("senderID"), 500)
        except gcmclient.GCMAuthenticationError, e:
            raise self._error("Authentication Error: %s" % e, 500)
        except Exception, e:
            raise self._error("Unhandled exception in GCM Routing: %s" % e,
                              500)
        self.metrics.increment("updates.client.bridge.gcm.attempted",
                               self._base_tags)
        return self._process_reply(result)

    def _error(self, err, status, **kwargs):
        """Error handler that raises the RouterException"""
        self.log.debug(err, **kwargs)
        return RouterException(err, status_code=status, response_body=err,
                               **kwargs)

    def _process_reply(self, reply):
        """Process GCM send reply"""
        # acks:
        #  for reg_id, msg_id in reply.success.items():
        # updates
        for old_id, new_id in reply.canonical.items():
            self.log.debug("GCM id changed : {old} => {new}",
                           old=old_id, new=new_id)
            return RouterResponse(status_code=503,
                                  response_body="Please try request again.",
                                  router_data=dict(token=new_id))
        # naks:
        # uninstall:
        for reg_id in reply.not_registered:
            self.log.debug("GCM no longer registered: %s" % reg_id)
            return RouterResponse(
                status_code=410,
                response_body="Endpoint requires client update",
                router_data={},
            )

        #  for reg_id, err_code in reply.failed.items():
        if len(reply.failed.items()) > 0:
            self.log.debug("GCM failures: {failed()}",
                           failed=lambda: json.dumps(reply.failed.items()))
            raise RouterException("GCM failure to deliver", status_code=503,
                                  response_body="Please try request later.")

        # retries:
        if reply.needs_retry():
            raise RouterException("GCM failure to deliver", status_code=503,
                                  response_body="Please try request later.")

        self.metrics.increment("updates.client.bridge.gcm.succeeded",
                               self._base_tags)
        return RouterResponse(status_code=200, response_body="Message Sent")
<commit_msg>bug: Elevate GCM reported error states<commit_after>"""GCM Router"""
import gcmclient
import json

from twisted.internet.threads import deferToThread
from twisted.logger import Logger

from autopush.router.interface import RouterException, RouterResponse
from autopush.senderids import SenderIDs

class GCMRouter(object):
    """GCM Router Implementation"""
    log = Logger()
    gcm = None
    dryRun = 0
    collapseKey = "simplepush"

    def __init__(self, ap_settings, router_conf):
        """Create a new GCM router and connect to GCM"""
        self.config = router_conf
        self.min_ttl = router_conf.get("ttl", 60)
        self.dryRun = router_conf.get("dryrun", False)
        self.collapseKey = router_conf.get("collapseKey", "simplepush")
        self.senderIDs = router_conf.get("senderIDs")
        self.metrics = ap_settings.metrics
        self._base_tags = []
        if not self.senderIDs:
            self.senderIDs = SenderIDs(router_conf)
        try:
            senderID = self.senderIDs.choose_ID()
            self.gcm = gcmclient.GCM(senderID.get("auth"))
        except:
            raise IOError("GCM Bridge not initiated in main")
        self.log.debug("Starting GCM router...")

    def check_token(self, token):
        if token not in self.senderIDs.senderIDs():
            return (False, self.senderIDs.choose_ID().get('senderID'))
        return (True, token)

    def amend_msg(self, msg, data=None):
        if data is not None:
            msg["senderid"] = data.get('creds', {}).get('senderID')
        return msg

    def register(self, uaid, router_data, router_token=None, *kwargs):
        """Validate that the GCM Instance Token is in the ``router_data``"""
        if "token" not in router_data:
            raise self._error("connect info missing GCM Instance 'token'",
                              status=401)
        # Assign a senderid
        router_data["creds"] = self.senderIDs.get_ID(router_token)
        return router_data

    def route_notification(self, notification, uaid_data):
        """Start the GCM notification routing, returns a deferred"""
        router_data = uaid_data["router_data"]
        # Kick the entire notification routing off to a thread
        return deferToThread(self._route, notification, router_data)

    def _route(self, notification, router_data):
        """Blocking GCM call to route the notification"""
        data = {"chid": notification.channel_id}
        # Payload data is optional. The endpoint handler validates that the
        # correct encryption headers are included with the data.
        if notification.data:
            mdata = self.config.get('max_data', 4096)
            if len(notification.data) > mdata:
                raise self._error("This message is intended for a " +
                                  "constrained device and is limited " +
                                  "to 3070 bytes. Converted buffer too " +
                                  "long by %d bytes" %
                                  (len(notification.data) - mdata),
                                  413, errno=104)

            data['body'] = notification.data
            data['con'] = notification.headers['content-encoding']
            data['enc'] = notification.headers['encryption']

            if 'crypto-key' in notification.headers:
                data['cryptokey'] = notification.headers['crypto-key']
            elif 'encryption-key' in notification.headers:
                data['enckey'] = notification.headers['encryption-key']

        # registration_ids are the GCM instance tokens (specified during
        # registration.
        router_ttl = notification.ttl or 0
        payload = gcmclient.JSONMessage(
            registration_ids=[router_data.get("token")],
            collapse_key=self.collapseKey,
            time_to_live=max(self.min_ttl, router_ttl),
            dry_run=self.dryRun or ("dryrun" in router_data),
            data=data,
        )
        creds = router_data.get("creds", {"senderID": "missing id"})
        try:
            self.gcm.api_key = creds["auth"]
            result = self.gcm.send(payload)
        except KeyError:
            raise self._error("Server error, missing bridge credentials " +
                              "for %s" % creds.get("senderID"), 500)
        except gcmclient.GCMAuthenticationError, e:
            raise self._error("Authentication Error: %s" % e, 500)
        except Exception, e:
            raise self._error("Unhandled exception in GCM Routing: %s" % e,
                              500)
        self.metrics.increment("updates.client.bridge.gcm.attempted",
                               self._base_tags)
        return self._process_reply(result)

    def _error(self, err, status, **kwargs):
        """Error handler that raises the RouterException"""
        self.log.debug(err, **kwargs)
        return RouterException(err, status_code=status, response_body=err,
                               **kwargs)

    def _process_reply(self, reply):
        """Process GCM send reply"""
        # acks:
        #  for reg_id, msg_id in reply.success.items():
        # updates
        for old_id, new_id in reply.canonical.items():
            self.log.info("GCM id changed : {old} => {new}",
                          old=old_id, new=new_id)
            self.metrics.increment("updates.client.bridge.gcm.failed.rereg",
                                   self._base_tags)
            return RouterResponse(status_code=503,
                                  response_body="Please try request again.",
                                  router_data=dict(token=new_id))
        # naks:
        # uninstall:
        for reg_id in reply.not_registered:
            self.metrics.increment("updates.client.bridge.gcm.failed.unreg",
                                   self._base_tags)
            self.log.info("GCM no longer registered: %s" % reg_id)
            return RouterResponse(
                status_code=410,
                response_body="Endpoint requires client update",
                router_data={},
            )

        #  for reg_id, err_code in reply.failed.items():
        if len(reply.failed.items()) > 0:
            self.metrics.increment("updates.client.bridge.gcm.failed.failure",
                                   self._base_tags)
            self.log.critical("GCM failures: {failed()}",
                              failed=lambda: json.dumps(reply.failed.items()))
            raise RouterException("GCM failure to deliver", status_code=503,
                                  response_body="Please try request later.")

        # retries:
        if reply.needs_retry():
            self.log.warn("GCM retry requested: {failed()}",
                          failed=lambda: json.dumps(reply.failed.items()))
            self.metrics.increment("updates.client.bridge.gcm.failed.retry",
                                   self._base_tags)
            raise RouterException("GCM failure to deliver, retry",
                                  status_code=503,
                                  response_body="Please try request later.")

        self.metrics.increment("updates.client.bridge.gcm.succeeded",
                               self._base_tags)
        return RouterResponse(status_code=200, response_body="Message Sent")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import subprocess

GIT_CMD = 'git log | head -3'

def git_stats():
  try:
    ret = subprocess.check_output([GIT_CMD], shell=True)\
      .decode('utf-8')\
      .split("\n")
    if len(ret) >= 3:
      commit = ret[0].split()[1]
      author = (" ").join(ret[1].split()[1:])
      date = (" ").join(ret[2].split()[1:])
      return {
        "commit": commit,
        "author": author,
        "date":   date
      }
    else:
      return {}
  except subprocess.CalledProcessError:
    return {}
<commit_msg>Switch to regex matching<commit_after># -*- coding: utf-8 -*-

import subprocess, re

GIT_CMD = 'git log | head -4'
RE_COMMIT = re.compile(r'commit[\W]+(\w+)', re.MULTILINE)
RE_AUTHOR = re.compile(r'Author\:[\W]+(.*)', re.MULTILINE)
RE_DATE = re.compile(r'Date\:[\W]+(.*)', re.MULTILINE)

def git_stats():
  try:
    ret = subprocess.check_output([GIT_CMD], shell=True)\
      .decode('utf-8')
    commit = re.search(RE_COMMIT, ret)
    author = re.search(RE_AUTHOR, ret)
    date = re.search(RE_DATE, ret)
    return {
      "commit": commit.group(1) if commit else "Unknown",
      "author": author.group(1) if author else "Unknown",
      "date":   date.group(1) if date else "Unknown"
    }
  except subprocess.CalledProcessError:
    return { "Error" }
<|endoftext|>