<commit_before># python3
# Copyright 2018 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Adders that use Reverb (github.com/deepmind/reverb) as a backend."""

import abc
from typing import Callable, Iterable, Mapping, NamedTuple, Optional, Sized, Union, Tuple

from absl import logging
from acme import specs
from acme import types
from acme.adders import base
import dm_env
import numpy as np
import reverb
import tensorflow as tf
import tree

DEFAULT_PRIORITY_TABLE = 'priority_table'
StartOfEpisodeType = Union[bool, specs.Array, tf.Tensor, tf.TensorSpec,
                           Tuple[()]]

# TODO(b/188510142): Delete Step.
class Step(NamedTuple):
  """Step class used internally for reverb adders."""
  observation: types.NestedArray
  action: types.NestedArray
  reward: types.NestedArray
  discount: types.NestedArray
  start_of_episode: StartOfEpisodeType
  extras: types.NestedArray = ()

# TODO(b/188510142): Replace with proper Trajectory class.
Trajectory = Step

class PriorityFnInput(NamedTuple):
  """The input to a priority function consisting of stacked steps."""
  observations: types.NestedArray
  actions: types.NestedArray
  rewards: types.NestedArray
  discounts: types.NestedArray
  start_of_episode: types.NestedArray
  extras: types.NestedArray

# Define the type of a priority function and the mapping from table to function.
PriorityFn = Callable[['PriorityFnInput'], float]
PriorityFnMapping = Mapping[str, Optional[PriorityFn]]

def spec_like_to_tensor_spec(paths: Iterable[str], spec: specs.Array):
  return tf.TensorSpec.from_spec(spec, name='/'.join(str(p) for p in paths))

class ReverbAdder(base.Adder):
  """Base class for Reverb adders."""

  def __init__(
      self,
      client: reverb.Client,
      max_sequence_length: int,
      max_in_flight_items: int,
      delta_encoded: bool = False,
      priority_fns: Optional[PriorityFnMapping] = None,
      get_signature_timeout_ms: int = 300_000,
  ):
    """Initialize a ReverbAdder instance.

    Args:
      client: A client to the Reverb backend.
      max_sequence_length: The maximum length of sequences (corresponding to the
        number of observations) that can be added to replay.
      max_in_flight_items: The maximum number of items allowed to be "in flight"
        at the same time. See `block_until_num_items` in
        `reverb.TrajectoryWriter.flush` for more info.
      delta_encoded: If `True` (False by default) enables delta encoding, see
        `Client` for more information.
      priority_fns: A mapping from table names to priority functions; if
        omitted, all transitions/steps/sequences are given uniform priorities
        (1.0) and placed in DEFAULT_PRIORITY_TABLE.
      get_signature_timeout_ms: time before timeout in fetching the signature
        from the reverb server.
    """
    if priority_fns:
      priority_fns = dict(priority_fns)
    else:
      priority_fns = {DEFAULT_PRIORITY_TABLE: None}

    self._client = client
    self._priority_fns = priority_fns
    self._max_sequence_length = max_sequence_length
    self._delta_encoded = delta_encoded
    self._max_in_flight_items = max_in_flight_items
    self._add_first_called = False

    # This is exposed as the _writer property in such a way that it will create
    # a new writer automatically whenever the internal __writer is None. Users
    # should ONLY ever interact with self._writer.
    self.__writer = None
    # Every time a new writer is created, it must fetch the signature from the
    # Reverb server. If this is set too low it can crash the adders in a
    # distributed setup where the replay may take a while to spin up.
    self._get_signature_timeout_ms = get_signature_timeout_ms

  def __del__(self):
    if self.__writer is not None:
      timeout_ms = 10_000
      # Try flush all appended data before closing to avoid loss of experience.
      try:
        self.__writer.flush(self._max_in_flight_items, timeout_ms=timeout_ms)
      except reverb.DeadlineExceededError as e:
        logging.error(
            'Timeout (%d ms) exceeded when flushing the writer before '
            'deleting it. Caught Reverb exception: %s', timeout_ms, str(e))
      self.__writer.close()

  @property
  def _writer(self) -> reverb.TrajectoryWriter:
    if self.__writer is None:
      self.__writer = self._client.trajectory_writer(
          num_keep_alive_refs=self._max_sequence_length,
          get_signature_timeout_ms=self._get_signature_timeout_ms)
    return self.__writer

  def add_priority_table(self, table_name: str,
                         priority_fn: Optional[PriorityFn]):
    if table_name in self._priority_fns:
      raise ValueError(
          f'A priority function already exists for {table_name}. '
          f'Existing tables: {", ".join(self._priority_fns.keys())}.'
      )
    self._priority_fns[table_name] = priority_fn

  def reset(self, timeout_ms: Optional[int] = None):
    """Resets the adder's buffer."""
    if self.__writer:
      # Flush all appended data and clear the buffers.
      self.__writer.end_episode(clear_buffers=True, timeout_ms=timeout_ms)
    self._add_first_called = False

  def add_first(self, timestep: dm_env.TimeStep):
    """Record the first observation of a trajectory."""
    if not timestep.first():
      raise ValueError('adder.add_first with an initial timestep (i.e. one for '
                       'which timestep.first() is True')

    # Record the next observation but leave the history buffer row open by
    # passing `partial_step=True`.
    self._writer.append(dict(observation=timestep.observation,
                             start_of_episode=timestep.first()),
                        partial_step=True)
    self._add_first_called = True

  def add(self,
          action: types.NestedArray,
          next_timestep: dm_env.TimeStep,
          extras: types.NestedArray = ()):
    """Record an action and the following timestep."""

    if not self._add_first_called:
      raise ValueError('adder.add_first must be called before adder.add.')

    # Add the timestep to the buffer.
    has_extras = (len(extras) > 0 if isinstance(extras, Sized)  # pylint: disable=g-explicit-length-test
                  else extras is not None)
    current_step = dict(
        # Observation was passed at the previous add call.
        action=action,
        reward=next_timestep.reward,
        discount=next_timestep.discount,
        # Start of episode indicator was passed at the previous add call.
        **({'extras': extras} if has_extras else {})
    )
    self._writer.append(current_step)

    # Record the next observation and write.
    self._writer.append(
        dict(
            observation=next_timestep.observation,
            start_of_episode=next_timestep.first()),
        partial_step=True)
    self._write()

    if next_timestep.last():
      # Complete the row by appending zeros to remaining open fields.
      # TODO(b/183945808): remove this when fields are no longer expected to be
      # of equal length on the learner side.
      dummy_step = tree.map_structure(np.zeros_like, current_step)
      self._writer.append(dummy_step)
      self._write_last()
      self.reset()

  @classmethod
  def signature(cls, environment_spec: specs.EnvironmentSpec,
                extras_spec: types.NestedSpec = ()):
    """This is a helper method for generating signatures for Reverb tables.

    Signatures are useful for validating data types and shapes, see Reverb's
    documentation for details on how they are used.

    Args:
      environment_spec: A `specs.EnvironmentSpec` whose fields are nested
        structures with leaf nodes that have `.shape` and `.dtype` attributes.
        This should come from the environment that will be used to generate
        the data inserted into the Reverb table.
      extras_spec: A nested structure with leaf nodes that have `.shape` and
        `.dtype` attributes. The structure (and shapes/dtypes) of this must
        be the same as the `extras` passed into `ReverbAdder.add`.

    Returns:
      A `Step` whose leaf nodes are `tf.TensorSpec` objects.
    """
    spec_step = Step(
        observation=environment_spec.observations,
        action=environment_spec.actions,
        reward=environment_spec.rewards,
        discount=environment_spec.discounts,
        start_of_episode=specs.Array(shape=(), dtype=bool),
        extras=extras_spec)
    return tree.map_structure_with_path(spec_like_to_tensor_spec, spec_step)

  @abc.abstractmethod
  def _write(self):
    """Write data to replay from the buffer."""

  @abc.abstractmethod
  def _write_last(self):
    """Write data to replay from the buffer."""
<commit_msg>Make adders create new writers regularly.<commit_after># python3
# Copyright 2018 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Adders that use Reverb (github.com/deepmind/reverb) as a backend."""

import abc
import time
from typing import Callable, Iterable, Mapping, NamedTuple, Optional, Sized, Union, Tuple

from absl import logging
from acme import specs
from acme import types
from acme.adders import base
import dm_env
import numpy as np
import reverb
import tensorflow as tf
import tree

DEFAULT_PRIORITY_TABLE = 'priority_table'
_MIN_WRITER_LIFESPAN_SECONDS = 60
StartOfEpisodeType = Union[bool, specs.Array, tf.Tensor, tf.TensorSpec,
                           Tuple[()]]

# TODO(b/188510142): Delete Step.
class Step(NamedTuple):
  """Step class used internally for reverb adders."""
  observation: types.NestedArray
  action: types.NestedArray
  reward: types.NestedArray
  discount: types.NestedArray
  start_of_episode: StartOfEpisodeType
  extras: types.NestedArray = ()

# TODO(b/188510142): Replace with proper Trajectory class.
Trajectory = Step

class PriorityFnInput(NamedTuple):
  """The input to a priority function consisting of stacked steps."""
  observations: types.NestedArray
  actions: types.NestedArray
  rewards: types.NestedArray
  discounts: types.NestedArray
  start_of_episode: types.NestedArray
  extras: types.NestedArray

# Define the type of a priority function and the mapping from table to function.
PriorityFn = Callable[['PriorityFnInput'], float]
PriorityFnMapping = Mapping[str, Optional[PriorityFn]]

def spec_like_to_tensor_spec(paths: Iterable[str], spec: specs.Array):
  return tf.TensorSpec.from_spec(spec, name='/'.join(str(p) for p in paths))

class ReverbAdder(base.Adder):
  """Base class for Reverb adders."""

  def __init__(
      self,
      client: reverb.Client,
      max_sequence_length: int,
      max_in_flight_items: int,
      delta_encoded: bool = False,
      priority_fns: Optional[PriorityFnMapping] = None,
      get_signature_timeout_ms: int = 300_000,
  ):
    """Initialize a ReverbAdder instance.

    Args:
      client: A client to the Reverb backend.
      max_sequence_length: The maximum length of sequences (corresponding to the
        number of observations) that can be added to replay.
      max_in_flight_items: The maximum number of items allowed to be "in flight"
        at the same time. See `block_until_num_items` in
        `reverb.TrajectoryWriter.flush` for more info.
      delta_encoded: If `True` (False by default) enables delta encoding, see
        `Client` for more information.
      priority_fns: A mapping from table names to priority functions; if
        omitted, all transitions/steps/sequences are given uniform priorities
        (1.0) and placed in DEFAULT_PRIORITY_TABLE.
      get_signature_timeout_ms: time before timeout in fetching the signature
        from the reverb server.
    """
    if priority_fns:
      priority_fns = dict(priority_fns)
    else:
      priority_fns = {DEFAULT_PRIORITY_TABLE: None}

    self._client = client
    self._priority_fns = priority_fns
    self._max_sequence_length = max_sequence_length
    self._delta_encoded = delta_encoded
    self._max_in_flight_items = max_in_flight_items
    self._add_first_called = False

    # This is exposed as the _writer property in such a way that it will create
    # a new writer automatically whenever the internal __writer is None. Users
    # should ONLY ever interact with self._writer.
    self.__writer = None
    # Every time a new writer is created, it must fetch the signature from the
    # Reverb server. If this is set too low it can crash the adders in a
    # distributed setup where the replay may take a while to spin up.
    self._get_signature_timeout_ms = get_signature_timeout_ms

  def __del__(self):
    if self.__writer is not None:
      timeout_ms = 10_000
      # Try flush all appended data before closing to avoid loss of experience.
      try:
        self.__writer.flush(self._max_in_flight_items, timeout_ms=timeout_ms)
      except reverb.DeadlineExceededError as e:
        logging.error(
            'Timeout (%d ms) exceeded when flushing the writer before '
            'deleting it. Caught Reverb exception: %s', timeout_ms, str(e))
      self.__writer.close()

  @property
  def _writer(self) -> reverb.TrajectoryWriter:
    if self.__writer is None:
      self.__writer = self._client.trajectory_writer(
          num_keep_alive_refs=self._max_sequence_length,
          get_signature_timeout_ms=self._get_signature_timeout_ms)
      self._writer_created_timestamp = time.time()
    return self.__writer

  def add_priority_table(self, table_name: str,
                         priority_fn: Optional[PriorityFn]):
    if table_name in self._priority_fns:
      raise ValueError(
          f'A priority function already exists for {table_name}. '
          f'Existing tables: {", ".join(self._priority_fns.keys())}.'
      )
    self._priority_fns[table_name] = priority_fn

  def reset(self, timeout_ms: Optional[int] = None):
    """Resets the adder's buffer."""
    if self.__writer:
      # Flush all appended data and clear the buffers.
      self.__writer.end_episode(clear_buffers=True, timeout_ms=timeout_ms)

      # Create a new writer unless the current one is too young.
      # This is to reduce the relative overhead of creating a new Reverb writer.
      if (time.time() - self._writer_created_timestamp >
          _MIN_WRITER_LIFESPAN_SECONDS):
        self.__writer = None
    self._add_first_called = False

  def add_first(self, timestep: dm_env.TimeStep):
    """Record the first observation of a trajectory."""
    if not timestep.first():
      raise ValueError('adder.add_first with an initial timestep (i.e. one for '
                       'which timestep.first() is True')

    # Record the next observation but leave the history buffer row open by
    # passing `partial_step=True`.
    self._writer.append(dict(observation=timestep.observation,
                             start_of_episode=timestep.first()),
                        partial_step=True)
    self._add_first_called = True

  def add(self,
          action: types.NestedArray,
          next_timestep: dm_env.TimeStep,
          extras: types.NestedArray = ()):
    """Record an action and the following timestep."""

    if not self._add_first_called:
      raise ValueError('adder.add_first must be called before adder.add.')

    # Add the timestep to the buffer.
    has_extras = (len(extras) > 0 if isinstance(extras, Sized)  # pylint: disable=g-explicit-length-test
                  else extras is not None)
    current_step = dict(
        # Observation was passed at the previous add call.
        action=action,
        reward=next_timestep.reward,
        discount=next_timestep.discount,
        # Start of episode indicator was passed at the previous add call.
        **({'extras': extras} if has_extras else {})
    )
    self._writer.append(current_step)

    # Record the next observation and write.
    self._writer.append(
        dict(
            observation=next_timestep.observation,
            start_of_episode=next_timestep.first()),
        partial_step=True)
    self._write()

    if next_timestep.last():
      # Complete the row by appending zeros to remaining open fields.
      # TODO(b/183945808): remove this when fields are no longer expected to be
      # of equal length on the learner side.
      dummy_step = tree.map_structure(np.zeros_like, current_step)
      self._writer.append(dummy_step)
      self._write_last()
      self.reset()

  @classmethod
  def signature(cls, environment_spec: specs.EnvironmentSpec,
                extras_spec: types.NestedSpec = ()):
    """This is a helper method for generating signatures for Reverb tables.

    Signatures are useful for validating data types and shapes, see Reverb's
    documentation for details on how they are used.

    Args:
      environment_spec: A `specs.EnvironmentSpec` whose fields are nested
        structures with leaf nodes that have `.shape` and `.dtype` attributes.
        This should come from the environment that will be used to generate
        the data inserted into the Reverb table.
      extras_spec: A nested structure with leaf nodes that have `.shape` and
        `.dtype` attributes. The structure (and shapes/dtypes) of this must
        be the same as the `extras` passed into `ReverbAdder.add`.

    Returns:
      A `Step` whose leaf nodes are `tf.TensorSpec` objects.
    """
    spec_step = Step(
        observation=environment_spec.observations,
        action=environment_spec.actions,
        reward=environment_spec.rewards,
        discount=environment_spec.discounts,
        start_of_episode=specs.Array(shape=(), dtype=bool),
        extras=extras_spec)
    return tree.map_structure_with_path(spec_like_to_tensor_spec, spec_step)

  @abc.abstractmethod
  def _write(self):
    """Write data to replay from the buffer."""

  @abc.abstractmethod
  def _write_last(self):
    """Write data to replay from the buffer."""
<|endoftext|>


<commit_before>"""Consume DOT REST Service with Iowa Winter Road Conditions

Run every five minutes from RUN_5MIN.sh

      "attributes" : {
        "OBJECTID" : 38,
        "SEGMENT_ID" : 1883,
        "ROUTE" : "I-35",
        "NAMEID" : "I-35: Swaledale Exit to Exit 194 (Clear Lake)",
        "LONGNAME" : "I-35,.... LAKE)",
        "DISTRICT" : "D2-  Hanlontown-Mason City",
        "SUBAREA" : "Hanlontown",
        "PRIMARYMP" : 182.5526287,
        "PRIMARYLAT" : 42.97979554,
        "PRIMARYLONG" : -93.34460851,
        "SECONDARYMP" : 194.004904,
        "SECONDARYLAT" : 43.145956,
        "SECONDARYLONG" : -93,
        "EVENT_ID" : "IASEG-187400",
        "EVENT_UPDATE" : 73,
        "HL_PAVEMENT_CONDITION" : "seasonal roadway conditions",
        "LOC_LINK_DIRECTION" : "both directions",
        "ROAD_CONDITION" : "Seasonal",
        "GIS_CREATION_DATE" : 1449769515000,
        "CARS_MSG_UPDATE_DATE" : 1449766817000,
        "CARS_MSG_INITIAL_DATE" : 1449766817000,
        "PH_PAVEMENT_CONDITION" : "",
        "SHAPE_Length" : 25598.2073258733
      },

"""
import json
import datetime
import sys
import psycopg2.extras
import shapefile
import requests
from pyiem.util import exponential_backoff
from pyiem import wellknowntext
import zipfile
import subprocess
import os

EPSG26915 = ('PROJCS["NAD_1983_UTM_Zone_15N",GEOGCS["GCS_North_American_1983"'
             ',DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,'
             '298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",'
             '0.0174532925199433]],PROJECTION["Transverse_Mercator"],'
             'PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",'
             '0.0],PARAMETER["Central_Meridian",-93.0],PARAMETER'
             '["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0]'
             ',UNIT["Meter",1.0]]')

def export_shapefile(txn, utc):
    """Export a Shapefile of Road Conditions"""
    os.chdir("/tmp")
    w = shapefile.Writer(shapefile.POLYLINE)
    w.field("SEGID", 'N', 6, 0)
    w.field("MAJOR", 'S', 10, 0)
    w.field("MINOR", 'S', 128, 0)
    w.field("US1", 'N', 4, 0)
    w.field("ST1", 'N', 4, 0)
    w.field("INT1", 'N', 4, 0)
    w.field("TYPE", 'N', 4, 0)
    w.field("VALID", 'S', 12, 0)
    w.field("COND_CODE", 'N', 4, 0)
    w.field("COND_TXT", 'S', 120, 0)
    w.field("BAN_TOW", 'S', 1, 0)
    w.field("LIM_VIS", 'S', 1, 0)

    txn.execute("""select b.*, c.*, ST_astext(b.geom) as bgeom from
         roads_base b, roads_current c WHERE b.segid = c.segid
         and valid is not null and b.geom is not null""")
    for row in txn:
        s = row["bgeom"]
        f = wellknowntext.convert_well_known_text(s)
        w.line(parts=f)
        w.record(row["segid"],
                 row["major"],
                 row["minor"],
                 row["us1"],
                 row["st1"],
                 row["int1"],
                 row["type"],
                 row["valid"].strftime("%Y%m%d%H%M"),
                 row["cond_code"],
                 row["raw"],
                 str(row["towing_prohibited"])[0],
                 str(row["limited_vis"])[0])

    w.save('iaroad_cond')
    z = zipfile.ZipFile("iaroad_cond.zip", 'w')
    z.write("iaroad_cond.shp")
    z.write("iaroad_cond.shx")
    z.write("iaroad_cond.dbf")
    o = open('iaroad_cond.prj', 'w')
    o.write(EPSG26915)
    o.close()
    z.write("iaroad_cond.prj")
    z.close()

    subprocess.call(("/home/ldm/bin/pqinsert -p 'zip ac %s "
                     "gis/shape/26915/ia/iaroad_cond.zip "
                     "GIS/iaroad_cond_%s.zip zip' iaroad_cond.zip"
                     "") % (utc.strftime("%Y%m%d%H%M"),
                            utc.strftime("%Y%m%d%H%M")), shell=True)

    for suffix in ['shp', 'shx', 'dbf', 'prj', 'zip']:
        os.unlink("iaroad_cond.%s" % (suffix,))

URI = ("http://iowadot.maps.arcgis.com/sharing/rest/content/items/"
       "5d6c7d6963e549539ead6e50d89bdd08/data")

pgconn = psycopg2.connect(database='postgis', host='iemdb')
cursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)

# NB: Could be 'null' (None) as well
ROADCOND = {
    'dry pavement': 0,
    'wet': 1,
    'partially covered with frost': 3,
    'partially covered with snow': 39,
    'partially covered with ice': 27,
    'partially covered with slush': 56,
    'partially covered with mix': 15,
    'completely covered with frost': 11,
    'completely covered with snow': 47,
    'completely covered with ice': 35,
    'completely covered with slush': 64,
    'completely covered with mixed': 23,
    'travel not advised': 51,
    'impassable': 86,
    'partially covered with mixed snow ice or slush': 15,
    'completely covered with mixed snow ice or slush': 23,
    'icy bridges': 27,
    'seasonal': 0,
    'Seasonal': 0,
    'seasonal roadway conditions': 0,
    'impassable': 86,
            }

lookup = {}
current = {}
cursor.execute("""
    select c.segid, b.longname, c.cond_code, b.idot_id from
    roads_current c JOIN roads_base b on (c.segid = b.segid)
    """)
for row in cursor:
    lookup[row[3]] = row[0]
    current[row[0]] = row[2]

r = exponential_backoff(requests.get, URI, timeout=30)
if r is None:
    sys.exit()
j = r.json()

if 'layers' not in j:
    print(('ingest_roads_rest got invalid RESULT:\n%s' % (
                json.dumps(j, sort_keys=True, indent=4, separators=(',', ': ')
                           ))))
    sys.exit()

featureset = j['layers'][0]['featureSet']
dirty = False
for feat in featureset['features']:
    props = feat['attributes']
    segid = lookup.get(props['SEGMENT_ID'])
    if segid is None:
        print("ingest_roads_rest unknown longname '%s' segment_id '%s'" % (
            props['LONGNAME'], props['SEGMENT_ID']))
        continue
    raw = props['HL_PAVEMENT_CONDITION']
    if raw is None:
        continue
    cond = ROADCOND.get(raw)
    if cond is None:
        print(("ingest_roads_reset longname '%s' has unknown cond '%s'\n%s"
               ) % (props['LONGNAME'],
                    props['HL_PAVEMENT_CONDITION'],
                    json.dumps(props, sort_keys=True,
                               indent=4, separators=(',', ': '))))
        continue
    if cond == current[segid]:
        continue
    # Timestamps appear to be UTC now
    valid = datetime.datetime(1970, 1, 1) + datetime.timedelta(
        seconds=props['CARS_MSG_UPDATE_DATE']/1000.)
    # Save to log
    cursor.execute("""INSERT into roads_2015_2016_log(segid, valid, cond_code,
    raw) VALUES (%s, %s, %s, %s)""", (segid,
                                      valid.strftime("%Y-%m-%d %H:%M+00"),
                                      cond, raw))
    # Update currents
    cursor.execute("""UPDATE roads_current SET cond_code = %s, valid = %s,
    raw = %s WHERE segid = %s
    """, (cond, valid.strftime("%Y-%m-%d %H:%M+00"), raw, segid))
    dirty = True

if dirty:
    export_shapefile(cursor, valid)

cursor.close()
pgconn.commit()
pgconn.close()
<commit_msg>good time to audit Iowa DOT winter roads REST ingest<commit_after>"""Consume DOT REST Service with Iowa Winter Road Conditions

Run every five minutes from RUN_5MIN.sh

      "attributes" : {
        "OBJECTID" : 38,
        "SEGMENT_ID" : 1883,
        "ROUTE" : "I-35",
        "NAMEID" : "I-35: Swaledale Exit to Exit 194 (Clear Lake)",
        "LONGNAME" : "I-35,.... LAKE)",
        "DISTRICT" : "D2-  Hanlontown-Mason City",
        "SUBAREA" : "Hanlontown",
        "PRIMARYMP" : 182.5526287,
        "PRIMARYLAT" : 42.97979554,
        "PRIMARYLONG" : -93.34460851,
        "SECONDARYMP" : 194.004904,
        "SECONDARYLAT" : 43.145956,
        "SECONDARYLONG" : -93,
        "EVENT_ID" : "IASEG-187400",
        "EVENT_UPDATE" : 73,
        "HL_PAVEMENT_CONDITION" : "seasonal roadway conditions",
        "LOC_LINK_DIRECTION" : "both directions",
        "ROAD_CONDITION" : "Seasonal",
        "GIS_CREATION_DATE" : 1449769515000,
        "CARS_MSG_UPDATE_DATE" : 1449766817000,
        "CARS_MSG_INITIAL_DATE" : 1449766817000,
        "PH_PAVEMENT_CONDITION" : "",
        "SHAPE_Length" : 25598.2073258733
      },

"""
from __future__ import print_function
import json
import datetime
import sys
import zipfile
import subprocess
import os

import psycopg2.extras
from shapely.wkb import loads
import shapefile
import requests
from pyiem.util import exponential_backoff

URI = ("https://iowadot.maps.arcgis.com/sharing/rest/content/items/"
       "5d6c7d6963e549539ead6e50d89bdd08/data")

EPSG26915 = ('PROJCS["NAD_1983_UTM_Zone_15N",GEOGCS["GCS_North_American_1983"'
             ',DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,'
             '298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",'
             '0.0174532925199433]],PROJECTION["Transverse_Mercator"],'
             'PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",'
             '0.0],PARAMETER["Central_Meridian",-93.0],PARAMETER'
             '["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0]'
             ',UNIT["Meter",1.0]]')
# NB: Could be 'null' (None) as well
ROADCOND = {
    'dry pavement': 0,
    'wet': 1,
    'partially covered with frost': 3,
    'partially covered with snow': 39,
    'partially covered with ice': 27,
    'partially covered with slush': 56,
    'partially covered with mix': 15,
    'completely covered with frost': 11,
    'completely covered with snow': 47,
    'completely covered with ice': 35,
    'completely covered with slush': 64,
    'completely covered with mixed': 23,
    'travel not advised': 51,
    'partially covered with mixed snow ice or slush': 15,
    'completely covered with mixed snow ice or slush': 23,
    'icy bridges': 27,
    'seasonal': 0,
    'Seasonal': 0,
    'seasonal roadway conditions': 0,
    'impassable': 86,
            }

def export_shapefile(txn, utc):
    """Export a Shapefile of Road Conditions"""
    os.chdir("/tmp")
    shp = shapefile.Writer(shapefile.POLYLINE)
    shp.field("SEGID", 'N', 6, 0)
    shp.field("MAJOR", 'S', 10, 0)
    shp.field("MINOR", 'S', 128, 0)
    shp.field("US1", 'N', 4, 0)
    shp.field("ST1", 'N', 4, 0)
    shp.field("INT1", 'N', 4, 0)
    shp.field("TYPE", 'N', 4, 0)
    shp.field("VALID", 'S', 12, 0)
    shp.field("COND_CODE", 'N', 4, 0)
    shp.field("COND_TXT", 'S', 120, 0)
    shp.field("BAN_TOW", 'S', 1, 0)
    shp.field("LIM_VIS", 'S', 1, 0)

    txn.execute("""
        select b.*, c.*, b.geom from
         roads_base b, roads_current c WHERE b.segid = c.segid
         and valid is not null and b.geom is not null
    """)
    for row in txn:
        multiline = loads(row["geom"].decode('hex'))
        shp.line(parts=[zip(*multiline.geoms[0].xy)])
        shp.record(row["segid"],
                   row["major"],
                   row["minor"],
                   row["us1"],
                   row["st1"],
                   row["int1"],
                   row["type"],
                   row["valid"].strftime("%Y%m%d%H%M"),
                   row["cond_code"],
                   row["raw"],
                   str(row["towing_prohibited"])[0],
                   str(row["limited_vis"])[0])

    shp.save('iaroad_cond')
    zfp = zipfile.ZipFile("iaroad_cond.zip", 'w')
    zfp.write("iaroad_cond.shp")
    zfp.write("iaroad_cond.shx")
    zfp.write("iaroad_cond.dbf")
    fp = open('iaroad_cond.prj', 'w')
    fp.write(EPSG26915)
    fp.close()
    zfp.write("iaroad_cond.prj")
    zfp.close()

    subprocess.call(("/home/ldm/bin/pqinsert -p 'zip ac %s "
                     "gis/shape/26915/ia/iaroad_cond.zip "
                     "GIS/iaroad_cond_%s.zip zip' iaroad_cond.zip"
                     "") % (utc.strftime("%Y%m%d%H%M"),
                            utc.strftime("%Y%m%d%H%M")), shell=True)

    for suffix in ['shp', 'shx', 'dbf', 'prj', 'zip']:
        os.unlink("iaroad_cond.%s" % (suffix,))

def main():
    """Go something greatish"""
    pgconn = psycopg2.connect(database='postgis', host='iemdb')
    cursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)

    lookup = {}
    current = {}
    cursor.execute("""
        select c.segid, b.longname, c.cond_code, b.idot_id from
        roads_current c JOIN roads_base b on (c.segid = b.segid)
        """)
    for row in cursor:
        lookup[row[3]] = row[0]
        current[row[0]] = row[2]

    req = exponential_backoff(requests.get, URI, timeout=30)
    if req is None:
        sys.exit()
    jobj = req.json()

    if 'layers' not in jobj:
        print(('ingest_roads_rest got invalid RESULT:\n%s'
               ) % (json.dumps(jobj, sort_keys=True, indent=4,
                               separators=(',', ': '))
                    ))
        sys.exit()

    featureset = jobj['layers'][0]['featureSet']
    dirty = False
    for feat in featureset['features']:
        props = feat['attributes']
        segid = lookup.get(props['SEGMENT_ID'])
        if segid is None:
            print(("ingest_roads_rest unknown longname '%s' segment_id '%s'"
                   ) % (props['LONGNAME'], props['SEGMENT_ID']))
            continue
        raw = props['HL_PAVEMENT_CONDITION']
        if raw is None:
            continue
        cond = ROADCOND.get(raw)
        if cond is None:
            print(("ingest_roads_reset longname '%s' has unknown cond '%s'\n%s"
                   ) % (props['LONGNAME'],
                        props['HL_PAVEMENT_CONDITION'],
                        json.dumps(props, sort_keys=True,
                                   indent=4, separators=(',', ': '))))
            continue
        if cond == current[segid]:
            continue
        # Timestamps appear to be UTC now
        valid = datetime.datetime(1970, 1, 1) + datetime.timedelta(
            seconds=props['CARS_MSG_UPDATE_DATE']/1000.)
        # Save to log
        cursor.execute("""
            INSERT into roads_2015_2016_log(segid, valid, cond_code, raw)
            VALUES (%s, %s, %s, %s)
        """, (segid, valid.strftime("%Y-%m-%d %H:%M+00"), cond, raw))
        # Update currents
        cursor.execute("""
            UPDATE roads_current SET cond_code = %s, valid = %s,
            raw = %s WHERE segid = %s
        """, (cond, valid.strftime("%Y-%m-%d %H:%M+00"), raw, segid))
        dirty = True

    if dirty:
        export_shapefile(cursor, valid)

    cursor.close()
    pgconn.commit()
    pgconn.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from pytest import raises
from robust.tools import retry, timeout, breaker
from robust.exception import (ContinuousFailureException,
                              TimeoutException,
                              ConnectionCutException)

class TestRetryCase(object):

    def test_happy_case(self):
        @retry(5)
        def dummy():
            return 42

        assert 42 == dummy()

    def test_fail_case(self):
        @retry(5)
        def fail():
            raise RuntimeError("Don't know what to do")

        with raises(ContinuousFailureException):
            fail()

    def test_callback(self):
        passed = False

        def do_pass():
            nonlocal passed
            passed = True

        @retry(5, do_pass)
        def fail():
            raise RuntimeError("Don't know what to do")

        do_pass()
        assert passed

class TestTimeoutCase(object):

    def test_happy_case(self):
        @timeout(1)
        def dummy():
            return 42

        assert 42 == dummy()

    def test_failure_case(self):
        @timeout(1)
        def fail():
            while True:
                pass

        with raises(TimeoutException):
            fail()

    def test_callback(self):
        passed = False

        def do_pass():
            nonlocal passed
            passed = True

        @timeout(1, do_pass)
        def fail():
            while True:
                pass

        do_pass()
        assert passed

class TestBreakerPattern(object):

    def test_cut_after_5_failures(self):

        @breaker(limit=5, revive=30)
        def fail():
            raise RuntimeError("Just failing for no good reason")

        with raises(ConnectionCutException):
            counter = 0
            for i in range(0, 5):
                fail()
                counter += 1

        assert counter == 5
<commit_msg>We expect Runtime Errors<commit_after>from pytest import raises
from robust.tools import retry, timeout, breaker
from robust.exception import (ContinuousFailureException,
                              TimeoutException,
                              ConnectionCutException)

class TestRetryCase(object):

    def test_happy_case(self):
        @retry(5)
        def dummy():
            return 42

        assert 42 == dummy()

    def test_fail_case(self):
        @retry(5)
        def fail():
            raise RuntimeError("Don't know what to do")

        with raises(ContinuousFailureException):
            fail()

    def test_callback(self):
        passed = False

        def do_pass():
            nonlocal passed
            passed = True

        @retry(5, do_pass)
        def fail():
            raise RuntimeError("Don't know what to do")

        do_pass()
        assert passed

class TestTimeoutCase(object):

    def test_happy_case(self):
        @timeout(1)
        def dummy():
            return 42

        assert 42 == dummy()

    def test_failure_case(self):
        @timeout(1)
        def fail():
            while True:
                pass

        with raises(TimeoutException):
            fail()

    def test_callback(self):
        passed = False

        def do_pass():
            nonlocal passed
            passed = True

        @timeout(1, do_pass)
        def fail():
            while True:
                pass

        do_pass()
        assert passed

class TestBreakerPattern(object):

    def test_cut_after_5_failures(self):

        @breaker(limit=5, revive=30)
        def fail():
            raise RuntimeError("Just failing for no good reason")

        with raises(ConnectionCutException):
            counter = 0
            for i in range(0, 5):
                try:
                    fail()
                except RuntimeError:
                    pass
                counter += 1

        assert counter == 5
<|endoftext|>


<commit_before>""" Plugin for server moderation

The bot will perform different tasks when some settings are enabled in a server:

_____________________________________NSFW Filter_____________________________________
    If enabled on the server, spots any text containing the keyword nsfw and a link.
    Then tries to delete their message, and post a link to the dedicated nsfw channel.

Commands:
    moderate
    mute
    unmute
    timeout
"""

from collections import defaultdict

import discord
import asyncio

from pcbot import Config, utils, Annotate
import plugins

moderate = Config("moderate", data=defaultdict(dict))
default_config = {}  # Used by add_setting helper function

def setup_default_config(server: discord.Server):
    """ Setup default settings for a server. """
    # Set to defaults if there is no config for the server
    if server.id not in moderate.data:
        moderate.data[server.id] = default_config
        moderate.save()
        return

    # Set to defaults if server's config is missing values
    if not all(k in moderate.data[server.id].keys() for k in default_config):
        moderate.data[server.id] = default_config
        moderate.save()

@plugins.command(name="moderate")
def moderate_(client, message, _: utils.placeholder):
    """ Change moderation settings. """
    pass

def add_setting(setting: str, default=True, name=None, permissions=None):
    """ Create a set of subcommands for the given setting (True or False).

    :param setting: display name for the setting.
    :param default: The default value for this setting.
    :param name: optionally set the name of the subcommand.
    :param permissions: what permissions are required to change this setting (list of str). """
    if not name:
        name = setting.lower().replace("\n", "").replace(" ", "")

    default_config[name] = default

    @moderate_.command(name=name, description="Display current {} setting.".format(setting), usage="[on | off]")
    def display(client: discord.Client, message: discord.Message):
        """ The command to display the current setting. """
        setup_default_config(message.server)
        current = moderate.data[message.server.id][name]
        yield from client.say(message, "{} is `{}`.".format(setting, "ON" if current else "OFF"))

    @display.command(name="on", hidden=True)
    @utils.permission(*permissions)
    def enable_setting(client: discord.Client, message: discord.Message):
        """ The command to enable this setting. """
        moderate.data[message.server.id][name] = True
        moderate.save()
        yield from client.say(message, "{} enabled.".format(setting))

    @display.command(name="off", hidden=True)
    @utils.permission(*permissions)
    def enable_setting(client: discord.Client, message: discord.Message):
        """ The command to enable this setting. """
        moderate.data[message.server.id][name] = False
        moderate.save()
        yield from client.say(message, "{} disabled.".format(setting))

add_setting("NSFW filter", permissions=["manage_server"])
add_setting("Changelog", permissions=["manage_server"])

@asyncio.coroutine
def manage_mute(client: discord.Client, message: discord.Message, function, *members: discord.Member):
    """ Add or remove Muted role for given members.

    :param function: either client.add_roles or client.remove_roles
    :return: list of muted/unmuted members or None """
    # Manage Roles is required to add or remove the Muted role
    assert message.server.me.permissions_in(message.channel).manage_roles, \
        "I need `Manage Roles` permission to use this command."

    muted_role = discord.utils.get(message.server.roles, name="Muted")

    # The server needs to properly manage the Muted role
    assert muted_role, "No role assigned for muting. Please create a `Muted` role."

    muted_members = []

    # Try giving members the Muted role
    for member in members:
        if member is message.server.me:
            yield from client.say(message, "I refuse to mute myself!")
            continue

        while True:
            try:
                yield from function(member, muted_role)
            except discord.errors.Forbidden:
                yield from client.say(message, "I do not have permission to give members the `Muted` role.")
                return None
            except discord.errors.HTTPException:
                continue
            else:
                muted_members.append(member)
                break

    return muted_members or None

@plugins.command(pos_check=True)
@utils.permission("manage_messages")
def mute(client: discord.Client, message: discord.Message, *members: Annotate.Member):
    """ Mute the specified members. """
    muted_members = yield from manage_mute(client, message, client.add_roles, *members)

    # Some members were muted, success!
    if muted_members:
        yield from client.say(message, "Muted {}".format(utils.format_members(*muted_members)))

@plugins.command(pos_check=True)
@utils.permission("manage_messages")
def unmute(client: discord.Client, message: discord.Message, *members: Annotate.Member):
    """ Unmute the specified members. """
    muted_members = yield from manage_mute(client, message, client.remove_roles, *members)

    # Some members were unmuted, success!
    if muted_members:
        yield from client.say(message, "Unmuted {}".format(utils.format_members(*muted_members)))

@plugins.command(pos_check=True)
@utils.permission("manage_messages")
def timeout(client: discord.Client, message: discord.Message, *members: Annotate.Member, minutes: int):
    """ Timeout the specified members for given minutes. """
    muted_members = yield from manage_mute(client, message, client.add_roles, *members)

    # Do not progress if the members were not successfully muted
    # At this point, manage_mute will have reported any errors
    if not muted_members:
        return

    yield from client.say(message, "Timed out {} for `{}` minutes.".format(utils.format_members(*muted_members),
                                                                           minutes))

    # Sleep for the given minutes and unmute the member
    yield from asyncio.sleep(minutes * 60)  # Since asyncio.sleep takes seconds, multiply by 60
    yield from manage_mute(client, message, client.remove_roles, *muted_members)

@asyncio.coroutine
def check_nsfw(client: discord.Client, message: discord.Message):
    """ Check if the message is NSFW (very rough check). """
    # Check if this server has nsfwfilter enabled
    if not moderate.data[message.server.id]["nsfwfilter"]:
        return False

    # Do not check if the channel is designed for nsfw content
    if "nsfw" in message.channel.name:
        return False

    # Check if message includes keyword nsfw and a link
    msg = message.content.lower()
    if "nsfw" in msg and ("http://" in msg or "https://" in msg):
        if message.server.me.permissions_in(message.channel).manage_messages:
            yield from client.delete_message(message)

        nsfw_channel = discord.utils.find(lambda c: "nsfw" in c.name, message.server.channels)

        if nsfw_channel:
            yield from client.say(message, "{0.mention}: **Please post NSFW content in {1.mention}**".format(
                message.author, nsfw_channel) )

        return True

@plugins.event()
def on_message(client: discord.Client, message: discord.Message):
    """ Check plugin settings. """
    # Do not check in private messages
    if message.channel.is_private:
        return False

    setup_default_config(message.server)

    nsfw_success = yield from check_nsfw(client, message)
    if nsfw_success is True:
        return True

def get_changelog_channel(server: discord.Server):
    """ Return the changelog channel for a server. """
    setup_default_config(server)
    if not moderate.data[server.id]["changelog"]:
        return

    channel = discord.utils.get(server.channels, name="changelog")
    if not channel:
        return

    permissions = channel.permissions_for(server.me)
    if not permissions.send_messages or not permissions.read_messages:
        return

    return channel

@plugins.event()
def on_message_delete(client: discord.Client, message: discord.Message):
    """ Update the changelog with deleted messages. """
    changelog_channel = get_changelog_channel(message.server)
    if not changelog_channel:
        return

    if message.channel == changelog_channel:
        return

    if message.author == client.user:
        return

    if message.content.startswith("|"):  # Custom check for pastas
        return

    yield from client.send_message(
        changelog_channel,
        "{0.author.mention}'s message was deleted in {0.channel.mention}:\n{0.clean_content}".format(message)
    )

@plugins.event()
def on_channel_create(client: discord.Client, channel: discord.Channel):
    """ Update the changelog with created channels. """
    if channel.is_private:
        return

    changelog_channel = get_changelog_channel(channel.server)
    if not changelog_channel:
        return

    # Differ between voice channels and text channels
    if channel.type == discord.ChannelType.text:
        yield from client.send_message(changelog_channel, "Channel {0.mention} was created.".format(channel))
    else:
        yield from client.send_message(changelog_channel, "Voice channel **{0.name}** was created.".format(channel))

@plugins.event()
def on_channel_delete(client: discord.Client, channel: discord.Channel):
    """ Update the changelog with deleted channels. """
    if channel.is_private:
        return

    changelog_channel = get_changelog_channel(channel.server)
    if not changelog_channel:
        return

    # Differ between voice channels and text channels
    if channel.type == discord.ChannelType.text:
        yield from client.send_message(changelog_channel, "Channel #{0.name} was deleted.".format(channel))
    else:
        yield from client.send_message(changelog_channel, "Voice channel **{0.name}** was deleted.".format(channel))

@plugins.event()
def on_member_join(client: discord.Client, member: discord.Member):
    """ Update the changelog with members joined. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel, "{0.mention} joined the server.".format(member))

@plugins.event()
def on_member_remove(client: discord.Client, member: discord.Member):
    """ Update the changelog with deleted channels. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel, "{0.mention} ({0.name}) left the server.".format(member))

@plugins.event()
def on_member_update(client: discord.Client, before: discord.Member, after: discord.Member):
    """ Update the changelog with any changed names. """
    name_change = not before.name == after.name
    nick_change = not before.nick == after.nick

    if not name_change and not nick_change:
        return

    changelog_channel = get_changelog_channel(after.server)
    if not changelog_channel:
        return

    # Format the nickname or username changed
    if name_change:
        m = "{0.mention} (previously **{0.name}**) changed their username to **{1.name}**."
    else:
        if not before.nick:
            m = "{0.mention} got the nickname **{1.nick}**."
        elif not after.nick:
            m = "{0.mention} (previously **{0.nick}**), no longer has a nickname."
        else:
            m = "{0.mention} (previously **{0.nick}**) got the nickname **{1.nick}**."

    yield from client.send_message(changelog_channel, m.format(before, after))

@plugins.event()
def on_member_ban(client: discord.Client, member: discord.Member):
    """ Update the changelog with banned members. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel,
                                   "{0.mention} ({0.name}) was banned from the server.".format(member))

@plugins.event()
def on_member_unban(client: discord.Client, member: discord.Member):
    """ Update the changelog with unbanned members. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel, "{0.mention} was unbanned from the server.".format(member))
<commit_msg>Disable changelogs for a server by default.<commit_after>""" Plugin for server moderation

The bot will perform different tasks when some settings are enabled in a server:

_____________________________________NSFW Filter_____________________________________
    If enabled on the server, spots any text containing the keyword nsfw and a link.
    Then tries to delete their message, and post a link to the dedicated nsfw channel.

Commands:
    moderate
    mute
    unmute
    timeout
"""

from collections import defaultdict

import discord
import asyncio

from pcbot import Config, utils, Annotate
import plugins

moderate = Config("moderate", data=defaultdict(dict))
default_config = {}  # Used by add_setting helper function

def setup_default_config(server: discord.Server):
    """ Setup default settings for a server. """
    # Set to defaults if there is no config for the server
    if server.id not in moderate.data:
        moderate.data[server.id] = default_config
        moderate.save()
        return

    # Set to defaults if server's config is missing values
    if not all(k in moderate.data[server.id].keys() for k in default_config):
        moderate.data[server.id] = default_config
        moderate.save()

@plugins.command(name="moderate")
def moderate_(client, message, _: utils.placeholder):
    """ Change moderation settings. """
    pass

def add_setting(setting: str, default=True, name=None, permissions=None):
    """ Create a set of subcommands for the given setting (True or False).

    :param setting: display name for the setting.
    :param default: The default value for this setting.
    :param name: optionally set the name of the subcommand.
    :param permissions: what permissions are required to change this setting (list of str). """
    if not name:
        name = setting.lower().replace("\n", "").replace(" ", "")

    default_config[name] = default

    @moderate_.command(name=name,
                       description="Display current {} setting or enable/disable it.".format(setting),
                       usage="[on | off]")
    def display_setting(client: discord.Client, message: discord.Message):
        """ The command to display the current setting. """
        setup_default_config(message.server)
        current = moderate.data[message.server.id][name]
        yield from client.say(message, "{} is **{}**.".format(setting, "enabled" if current else "disabled"))

    @display_setting.command(hidden=True)
    @utils.permission(*permissions)
    def on(client: discord.Client, message: discord.Message):
        """ The command to enable this setting. """
        moderate.data[message.server.id][name] = True
        moderate.save()
        yield from client.say(message, "{} **enabled**.".format(setting))

    @display_setting.command(hidden=True)
    @utils.permission(*permissions)
    def off(client: discord.Client, message: discord.Message):
        """ The command to enable this setting. """
        moderate.data[message.server.id][name] = False
        moderate.save()
        yield from client.say(message, "{} **disabled**.".format(setting))

add_setting("NSFW filter", permissions=["manage_server"])
add_setting("Changelog", permissions=["manage_server"], default=False)

@asyncio.coroutine
def manage_mute(client: discord.Client, message: discord.Message, function, *members: discord.Member):
    """ Add or remove Muted role for given members.

    :param function: either client.add_roles or client.remove_roles
    :return: list of muted/unmuted members or None """
    # Manage Roles is required to add or remove the Muted role
    assert message.server.me.permissions_in(message.channel).manage_roles, \
        "I need `Manage Roles` permission to use this command."

    muted_role = discord.utils.get(message.server.roles, name="Muted")

    # The server needs to properly manage the Muted role
    assert muted_role, "No role assigned for muting. Please create a `Muted` role."

    muted_members = []

    # Try giving members the Muted role
    for member in members:
        if member is message.server.me:
            yield from client.say(message, "I refuse to mute myself!")
            continue

        while True:
            try:
                yield from function(member, muted_role)
            except discord.errors.Forbidden:
                yield from client.say(message, "I do not have permission to give members the `Muted` role.")
                return None
            except discord.errors.HTTPException:
                continue
            else:
                muted_members.append(member)
                break

    return muted_members or None

@plugins.command(pos_check=True)
@utils.permission("manage_messages")
def mute(client: discord.Client, message: discord.Message, *members: Annotate.Member):
    """ Mute the specified members. """
    muted_members = yield from manage_mute(client, message, client.add_roles, *members)

    # Some members were muted, success!
    if muted_members:
        yield from client.say(message, "Muted {}".format(utils.format_members(*muted_members)))

@plugins.command(pos_check=True)
@utils.permission("manage_messages")
def unmute(client: discord.Client, message: discord.Message, *members: Annotate.Member):
    """ Unmute the specified members. """
    muted_members = yield from manage_mute(client, message, client.remove_roles, *members)

    # Some members were unmuted, success!
    if muted_members:
        yield from client.say(message, "Unmuted {}".format(utils.format_members(*muted_members)))

@plugins.command(pos_check=True)
@utils.permission("manage_messages")
def timeout(client: discord.Client, message: discord.Message, *members: Annotate.Member, minutes: int):
    """ Timeout the specified members for given minutes. """
    muted_members = yield from manage_mute(client, message, client.add_roles, *members)

    # Do not progress if the members were not successfully muted
    # At this point, manage_mute will have reported any errors
    if not muted_members:
        return

    yield from client.say(message, "Timed out {} for `{}` minutes.".format(utils.format_members(*muted_members),
                                                                           minutes))

    # Sleep for the given minutes and unmute the member
    yield from asyncio.sleep(minutes * 60)  # Since asyncio.sleep takes seconds, multiply by 60
    yield from manage_mute(client, message, client.remove_roles, *muted_members)

@asyncio.coroutine
def check_nsfw(client: discord.Client, message: discord.Message):
    """ Check if the message is NSFW (very rough check). """
    # Check if this server has nsfwfilter enabled
    if not moderate.data[message.server.id]["nsfwfilter"]:
        return False

    # Do not check if the channel is designed for nsfw content
    if "nsfw" in message.channel.name:
        return False

    # Check if message includes keyword nsfw and a link
    msg = message.content.lower()
    if "nsfw" in msg and ("http://" in msg or "https://" in msg):
        if message.server.me.permissions_in(message.channel).manage_messages:
            yield from client.delete_message(message)

        nsfw_channel = discord.utils.find(lambda c: "nsfw" in c.name, message.server.channels)

        if nsfw_channel:
            yield from client.say(message, "{0.mention}: **Please post NSFW content in {1.mention}**".format(
                message.author, nsfw_channel) )

        return True

@plugins.event()
def on_message(client: discord.Client, message: discord.Message):
    """ Check plugin settings. """
    # Do not check in private messages
    if message.channel.is_private:
        return False

    setup_default_config(message.server)

    nsfw_success = yield from check_nsfw(client, message)
    if nsfw_success is True:
        return True

def get_changelog_channel(server: discord.Server):
    """ Return the changelog channel for a server. """
    setup_default_config(server)
    if not moderate.data[server.id]["changelog"]:
        return

    channel = discord.utils.get(server.channels, name="changelog")
    if not channel:
        return

    permissions = channel.permissions_for(server.me)
    if not permissions.send_messages or not permissions.read_messages:
        return

    return channel

@plugins.event()
def on_message_delete(client: discord.Client, message: discord.Message):
    """ Update the changelog with deleted messages. """
    changelog_channel = get_changelog_channel(message.server)
    if not changelog_channel:
        return

    if message.channel == changelog_channel:
        return

    if message.author == client.user:
        return

    if message.content.startswith("|"):  # Custom check for pastas
        return

    yield from client.send_message(
        changelog_channel,
        "{0.author.mention}'s message was deleted in {0.channel.mention}:\n{0.clean_content}".format(message)
    )

@plugins.event()
def on_channel_create(client: discord.Client, channel: discord.Channel):
    """ Update the changelog with created channels. """
    if channel.is_private:
        return

    changelog_channel = get_changelog_channel(channel.server)
    if not changelog_channel:
        return

    # Differ between voice channels and text channels
    if channel.type == discord.ChannelType.text:
        yield from client.send_message(changelog_channel, "Channel {0.mention} was created.".format(channel))
    else:
        yield from client.send_message(changelog_channel, "Voice channel **{0.name}** was created.".format(channel))

@plugins.event()
def on_channel_delete(client: discord.Client, channel: discord.Channel):
    """ Update the changelog with deleted channels. """
    if channel.is_private:
        return

    changelog_channel = get_changelog_channel(channel.server)
    if not changelog_channel:
        return

    # Differ between voice channels and text channels
    if channel.type == discord.ChannelType.text:
        yield from client.send_message(changelog_channel, "Channel #{0.name} was deleted.".format(channel))
    else:
        yield from client.send_message(changelog_channel, "Voice channel **{0.name}** was deleted.".format(channel))

@plugins.event()
def on_member_join(client: discord.Client, member: discord.Member):
    """ Update the changelog with members joined. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel, "{0.mention} joined the server.".format(member))

@plugins.event()
def on_member_remove(client: discord.Client, member: discord.Member):
    """ Update the changelog with deleted channels. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel, "{0.mention} ({0.name}) left the server.".format(member))

@plugins.event()
def on_member_update(client: discord.Client, before: discord.Member, after: discord.Member):
    """ Update the changelog with any changed names. """
    name_change = not before.name == after.name
    nick_change = not before.nick == after.nick

    if not name_change and not nick_change:
        return

    changelog_channel = get_changelog_channel(after.server)
    if not changelog_channel:
        return

    # Format the nickname or username changed
    if name_change:
        m = "{0.mention} (previously **{0.name}**) changed their username to **{1.name}**."
    else:
        if not before.nick:
            m = "{0.mention} got the nickname **{1.nick}**."
        elif not after.nick:
            m = "{0.mention} (previously **{0.nick}**), no longer has a nickname."
        else:
            m = "{0.mention} (previously **{0.nick}**) got the nickname **{1.nick}**."

    yield from client.send_message(changelog_channel, m.format(before, after))

@plugins.event()
def on_member_ban(client: discord.Client, member: discord.Member):
    """ Update the changelog with banned members. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel,
                                   "{0.mention} ({0.name}) was banned from the server.".format(member))

@plugins.event()
def on_member_unban(client: discord.Client, member: discord.Member):
    """ Update the changelog with unbanned members. """
    changelog_channel = get_changelog_channel(member.server)
    if not changelog_channel:
        return

    yield from client.send_message(changelog_channel, "{0.mention} was unbanned from the server.".format(member))
<|endoftext|>


<commit_before>from django.conf import settings
from uuid import uuid1

class Registry(object):
    def __init__(self):
        self.items = {}

    def add(self, id, title, description, url):
        self.items[id] = dict(title=title, description=description, url=url)

    def get_configured_list(self):
        items = []
        for id in settings.DEVILRY_QUALIFIESFOREXAM_PLUGINS:
            item = self.items[id]
            items.append({
                'id': id,
                'title': unicode(item['title']),
                'description': unicode(item['description']),
                'pluginsessionid': 'test',
#                'pluginsessionid': uuid1().hex,
                'url': str(item['url'])
            })
        return items

qualifiesforexam_plugins = Registry()<commit_msg>devilry_qualifiesforexam.registry: Use uuid1 for pluginsessionid.<commit_after>from django.conf import settings
from uuid import uuid1

class Registry(object):
    def __init__(self):
        self.items = {}

    def add(self, id, title, description, url):
        self.items[id] = dict(title=title, description=description, url=url)

    def get_configured_list(self):
        items = []
        for id in settings.DEVILRY_QUALIFIESFOREXAM_PLUGINS:
            item = self.items[id]
            items.append({
                'id': id,
                'title': unicode(item['title']),
                'description': unicode(item['description']),
#                'pluginsessionid': 'test',
                'pluginsessionid': uuid1().hex,
                'url': str(item['url'])
            })
        return items

qualifiesforexam_plugins = Registry()<|endoftext|>


<commit_before>from ansible.plugins.action import ActionBase
from ansible.utils.vars import merge_hash

class ActionModule(ActionBase):

    def run(self, tmp=None, task_vars=None):
        results = super(ActionModule, self).run(tmp, task_vars)

        # copy our egg
        tmp = self._make_tmp_path()
        source_full = self._loader.get_real_file("falafel-1.35.0-py2.7.egg")
        tmp_src = self._connection._shell.join_path(tmp, 'insights')
        remote_path = self._transfer_file(source_full, tmp_src)

        results = merge_hash(results, self._execute_module(module_args={"egg_path": remote_path}, module_name="insights", tmp=tmp, task_vars=task_vars))
        return results
<commit_msg>Update action plugin to fix _make_tmp_path issue<commit_after>from ansible.plugins.action import ActionBase
from ansible.utils.vars import merge_hash

class ActionModule(ActionBase):

    def run(self, tmp=None, task_vars=None):
        results = super(ActionModule, self).run(tmp, task_vars)
        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user

        # copy our egg
        tmp = self._make_tmp_path(remote_user)
        source_full = self._loader.get_real_file("falafel-1.35.0-py2.7.egg")
        tmp_src = self._connection._shell.join_path(tmp, 'insights')
        remote_path = self._transfer_file(source_full, tmp_src)

        results = merge_hash(results, self._execute_module(module_args={"egg_path": remote_path}, module_name="insights", tmp=tmp, task_vars=task_vars))
        return results
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Estimates capacity needs for all builders of a given master.
"""

import argparse
import datetime
import json
import os
import subprocess
import sys
import tempfile
import urllib
import urllib2

import numpy

BASE_DIR = os.path.dirname(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

def get_builds(mastername, buildername, days):
  results = []
  for day in days:
    cursor = ''
    while True:
      response = urllib2.urlopen(
          'https://chrome-build-extract.appspot.com/get_builds'
          '?master=%s&builder=%s&day=%s&cursor=%s' % (
              urllib.quote(mastername),
              urllib.quote(buildername),
              urllib.quote(str(day)),
              urllib.quote(cursor)))
      data = json.load(response)
      results.extend(data['builds'])
      cursor = data['cursor']
      if not cursor:
        break

  return results

def estimate_capacity(builds):
  hourly_buckets = {}
  daily_buckets = {}
  build_times = []
  for build in builds:
    build_time = build['times'][1] - build['times'][0]
    build_times.append(build_time)

    changes = build['sourceStamp']['changes']
    if changes:
      assert len(changes) == 1
      timestamp = datetime.datetime.utcfromtimestamp(changes[0]['when'])
    else:
      # Fallback for builds that don't have blamelist/source stamp,
      # e.g. win_pgo.
      timestamp = datetime.datetime.utcfromtimestamp(build['times'][0])

    hourly_bucket = timestamp.strftime('%Y-%m-%d-%H')
    hourly_buckets.setdefault(hourly_bucket, []).append(build_time)

    daily_bucket = timestamp.strftime('%Y-%m-%d')
    daily_buckets.setdefault(daily_bucket, []).append(build_time)

  def min_bots(buckets, resolution_s):
    result = 0
    for times in buckets.itervalues():
      total_time = sum(times)
      bots = total_time / resolution_s
      result = max(result, bots)
    return result

  return {
    'hourly_bots': min_bots(hourly_buckets, 3600),
    'daily_bots': min_bots(daily_buckets, 3600 * 24),
    'build_times_s': build_times,
  }

def main(argv):
  parser = argparse.ArgumentParser()
  parser.add_argument('master')
  parser.add_argument('--days', type=int, default=14)
  parser.add_argument('--filter-by-blamelist')
  parser.add_argument('--filter-by-patch-project')

  args = parser.parse_args(argv)

  with tempfile.NamedTemporaryFile() as f:
    subprocess.check_call([
        os.path.join(BASE_DIR, 'scripts', 'tools', 'runit.py'),
        os.path.join(BASE_DIR, 'scripts', 'tools', 'dump_master_cfg.py'),
        'masters/%s' % args.master,
        f.name])
    master_config = json.load(f)

  builder_pools = []
  for builder in master_config['builders']:
    slave_set = set(builder['slavenames'])
    builddir = builder.get('slavebuilddir', builder['name'])
    for pool in builder_pools:
      if pool['slave_set'] == slave_set:
        pool['builders'].setdefault(builddir, []).append(builder['name'])
        break
    else:
      builder_pools.append({
        'slave_set': slave_set,
        'builders': {builddir: [builder['name']]},
      })

  days = []
  for i in range(args.days):
    days.append(datetime.date.today() - datetime.timedelta(days=(i + 1)))
  for index, pool in enumerate(builder_pools):
    print 'Pool #%d:' % (index + 1)
    pool_capacity = {
      'hourly_bots': 0.0,
      'daily_bots': 0.0,
    }
    # TODO(phajdan.jr): use multiprocessing pool to speed this up.
    for builddir, builders in pool['builders'].iteritems():
      print '  builddir "%s":' % builddir
      for builder in builders:
        raw_builds = get_builds(
            args.master.replace('master.', ''), builder, days)

        builds = []
        for build in raw_builds:
          properties = {p[0]: p[1] for p in build.get('properties', [])}
          if (args.filter_by_patch_project and
              properties.get('patch_project') != args.filter_by_patch_project):
            continue

          blamelist = build.get('blame', [])
          if (args.filter_by_blamelist and
              args.filter_by_blamelist not in blamelist):
            continue

          builds.append(build)

        capacity = estimate_capacity(builds)
        for key in ('hourly_bots', 'daily_bots'):
          pool_capacity[key] += capacity[key]

        if capacity['build_times_s']:
          avg_build_time = str(datetime.timedelta(
              seconds=int(numpy.mean(capacity['build_times_s']))))
          median_build_time = str(datetime.timedelta(
              seconds=int(numpy.median(capacity['build_times_s']))))
        else:
          avg_build_time = 'n/a'
          median_build_time = 'n/a'

        print '    %-45s %-9s %-9s %5.1f %5.1f' % (
            builder,
            avg_build_time,
            median_build_time,
            capacity['daily_bots'],
            capacity['hourly_bots'])
    print '  %-45s                       %5.1f %5.1f %5.1f' % (
        'total',
        pool_capacity['daily_bots'],
        pool_capacity['hourly_bots'],
        len(pool['slave_set']))

  return 0

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<commit_msg>Add an option to print matched builds to estimate_capacity.py<commit_after>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Estimates capacity needs for all builders of a given master.
"""

import argparse
import datetime
import json
import os
import subprocess
import sys
import tempfile
import urllib
import urllib2

import numpy

BASE_DIR = os.path.dirname(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

def get_builds(mastername, buildername, days):
  results = []
  for day in days:
    cursor = ''
    while True:
      response = urllib2.urlopen(
          'https://chrome-build-extract.appspot.com/get_builds'
          '?master=%s&builder=%s&day=%s&cursor=%s' % (
              urllib.quote(mastername),
              urllib.quote(buildername),
              urllib.quote(str(day)),
              urllib.quote(cursor)))
      data = json.load(response)
      results.extend(data['builds'])
      cursor = data['cursor']
      if not cursor:
        break

  return results

def estimate_capacity(builds):
  hourly_buckets = {}
  daily_buckets = {}
  build_times = []
  for build in builds:
    build_time = build['times'][1] - build['times'][0]
    build_times.append(build_time)

    changes = build['sourceStamp']['changes']
    if changes:
      assert len(changes) == 1
      timestamp = datetime.datetime.utcfromtimestamp(changes[0]['when'])
    else:
      # Fallback for builds that don't have blamelist/source stamp,
      # e.g. win_pgo.
      timestamp = datetime.datetime.utcfromtimestamp(build['times'][0])

    hourly_bucket = timestamp.strftime('%Y-%m-%d-%H')
    hourly_buckets.setdefault(hourly_bucket, []).append(build_time)

    daily_bucket = timestamp.strftime('%Y-%m-%d')
    daily_buckets.setdefault(daily_bucket, []).append(build_time)

  def min_bots(buckets, resolution_s):
    result = 0
    for times in buckets.itervalues():
      total_time = sum(times)
      bots = total_time / resolution_s
      result = max(result, bots)
    return result

  return {
    'hourly_bots': min_bots(hourly_buckets, 3600),
    'daily_bots': min_bots(daily_buckets, 3600 * 24),
    'build_times_s': build_times,
  }

def main(argv):
  parser = argparse.ArgumentParser()
  parser.add_argument('master')
  parser.add_argument('--days', type=int, default=14)
  parser.add_argument('--filter-by-blamelist')
  parser.add_argument('--filter-by-patch-project')
  parser.add_argument('--print-builds', action='store_true')

  args = parser.parse_args(argv)

  with tempfile.NamedTemporaryFile() as f:
    subprocess.check_call([
        os.path.join(BASE_DIR, 'scripts', 'tools', 'runit.py'),
        os.path.join(BASE_DIR, 'scripts', 'tools', 'dump_master_cfg.py'),
        'masters/%s' % args.master,
        f.name])
    master_config = json.load(f)

  builder_pools = []
  for builder in master_config['builders']:
    slave_set = set(builder['slavenames'])
    builddir = builder.get('slavebuilddir', builder['name'])
    for pool in builder_pools:
      if pool['slave_set'] == slave_set:
        pool['builders'].setdefault(builddir, []).append(builder['name'])
        break
    else:
      builder_pools.append({
        'slave_set': slave_set,
        'builders': {builddir: [builder['name']]},
      })

  days = []
  for i in range(args.days):
    days.append(datetime.date.today() - datetime.timedelta(days=(i + 1)))
  for index, pool in enumerate(builder_pools):
    print 'Pool #%d:' % (index + 1)
    pool_capacity = {
      'hourly_bots': 0.0,
      'daily_bots': 0.0,
    }
    # TODO(phajdan.jr): use multiprocessing pool to speed this up.
    for builddir, builders in pool['builders'].iteritems():
      print '  builddir "%s":' % builddir
      for builder in builders:
        raw_builds = get_builds(
            args.master.replace('master.', ''), builder, days)

        builds = []
        for build in raw_builds:
          properties = {p[0]: p[1] for p in build.get('properties', [])}
          if (args.filter_by_patch_project and
              properties.get('patch_project') != args.filter_by_patch_project):
            continue

          blamelist = build.get('blame', [])
          if (args.filter_by_blamelist and
              args.filter_by_blamelist not in blamelist):
            continue

          builds.append(build)

        capacity = estimate_capacity(builds)
        for key in ('hourly_bots', 'daily_bots'):
          pool_capacity[key] += capacity[key]

        if capacity['build_times_s']:
          avg_build_time = str(datetime.timedelta(
              seconds=int(numpy.mean(capacity['build_times_s']))))
          median_build_time = str(datetime.timedelta(
              seconds=int(numpy.median(capacity['build_times_s']))))
        else:
          avg_build_time = 'n/a'
          median_build_time = 'n/a'

        print '    %-45s %-9s %-9s %5.1f %5.1f' % (
            builder,
            avg_build_time,
            median_build_time,
            capacity['daily_bots'],
            capacity['hourly_bots'])
        if args.print_builds:
          for build in builds:
            build_time = build['times'][1] - build['times'][0]
            print '        build #%-9s %-9s' % (
                build['number'],
                str(datetime.timedelta(seconds=build_time)))
    print '  %-45s                       %5.1f %5.1f %5.1f' % (
        'total',
        pool_capacity['daily_bots'],
        pool_capacity['hourly_bots'],
        len(pool['slave_set']))

  return 0

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before># the notebook plugin
# handles notebook creation

import dateutil.parser
import logging

import json
import re

from docx import Document
from espresso.main import robot
from tinydb import where

ANNOUNCEMENT_REGEX = r'(?is)Announcement for (?P<date>\d+/\d+/\d+): (?P<announcement>.*)'

@robot.hear(ANNOUNCEMENT_REGEX)
def got_announcement(res):
    date = dateutil.parser.parse(res.match.group('date'))
    announcement = res.match.group('announcement')
    user = res.msg.user.name

    logging.debug("Got Announcement for date %s: %s", date, announcement)
    res.robot.brain.db.insert({"plugin": "notebook", "type": "announcement",
        "date": date.isoformat(), "announcement": announcement,
        "user": user, "channel": res.msg.channel.name})

@robot.respond(r'(?i)backfill announcements for (?P<date>\d+/\d+/\d+)')
def backfill_announcements(res):
    target_date = dateutil.parser.parse(res.match.group('date'))

    channel_history = json.loads(res.robot.slack_client.api_call('channels.history',
        channel=res.robot.slack_client.server.channels.find("announcements").id,
        inclusive=1))
    channel_message_type_events = channel_history['messages']
    channel_messages = filter(lambda m: ((m.get('type') == 'message') and ('subtype' not in m)), channel_message_type_events)

    for m in channel_messages:
        match = re.search(ANNOUNCEMENT_REGEX, m['text'])
        if match:
            date = dateutil.parser.parse(match.group('date'))
            announcement = match.group('announcement')
            user = res.robot.slack_client.server.users.find(m['user']).name
            if date == target_date:
                logging.debug("Got Announcement for date %s by user %s: %s",
                              date, user, announcement)
                res.robot.brain.db.insert({"plugin": "notebook", "type": "announcement",
                    "date": date.isoformat(), "announcement": announcement,
                    "user": user, "channel": 'announcements'})

@robot.respond(r'(?i)make a (new )?notebook (entry|template) for (?P<date>\d+/\d+/\d+)')
def make_entry(res):
    date = dateutil.parser.parse(res.match.group('date'))
    logging.debug("date: %s", date)
    logging.debug("db dump: %s", res.robot.brain.db.all())
    announcements = res.robot.brain.db.search((where('plugin') == 'notebook')
        & (where('type') == 'announcement')
        & (where('date') == date.isoformat())
        )

    logging.debug("announcements are %s", announcements)

    if announcements != []:
        document = Document()
        document.add_page_break()
        document.add_heading('{date}, the BEC'.format(date=date.strftime('%m/%d/%Y')), level=1)
        document.add_paragraph('Present team members: <enter them here>')
        document.add_heading('Announcements:', level=2)

        users = []
        for announcement in announcements:
            if announcement['user'] not in users:
                users.append(announcement['user'])

        logging.debug("users are %s", users)

        for user in sorted(users):
            real_name = res.robot.slack_client.server.users.find(user).real_name
            logging.debug("announcing user %s is %s", user, real_name)
            document.add_heading("{}:".format(real_name), level=3)
            for announcement in announcements:
                if announcement['user'] == user:
                    document.add_paragraph("{}".format(announcement['announcement']),
                                           style='ListBullet')

        document.save('test.docx')
<commit_msg>notebook.py: Back to a paragraph for "people" headings<commit_after># the notebook plugin
# handles notebook creation

import dateutil.parser
import logging

import json
import re

from docx import Document
from espresso.main import robot
from tinydb import where

ANNOUNCEMENT_REGEX = r'(?is)Announcement for (?P<date>\d+/\d+/\d+): (?P<announcement>.*)'

@robot.hear(ANNOUNCEMENT_REGEX)
def got_announcement(res):
    date = dateutil.parser.parse(res.match.group('date'))
    announcement = res.match.group('announcement')
    user = res.msg.user.name

    logging.debug("Got Announcement for date %s: %s", date, announcement)
    res.robot.brain.db.insert({"plugin": "notebook", "type": "announcement",
        "date": date.isoformat(), "announcement": announcement,
        "user": user, "channel": res.msg.channel.name})

@robot.respond(r'(?i)backfill announcements for (?P<date>\d+/\d+/\d+)')
def backfill_announcements(res):
    target_date = dateutil.parser.parse(res.match.group('date'))

    channel_history = json.loads(res.robot.slack_client.api_call('channels.history',
        channel=res.robot.slack_client.server.channels.find("announcements").id,
        inclusive=1))
    channel_message_type_events = channel_history['messages']
    channel_messages = filter(lambda m: ((m.get('type') == 'message') and ('subtype' not in m)), channel_message_type_events)

    for m in channel_messages:
        match = re.search(ANNOUNCEMENT_REGEX, m['text'])
        if match:
            date = dateutil.parser.parse(match.group('date'))
            announcement = match.group('announcement')
            user = res.robot.slack_client.server.users.find(m['user']).name
            if date == target_date:
                logging.debug("Got Announcement for date %s by user %s: %s",
                              date, user, announcement)
                res.robot.brain.db.insert({"plugin": "notebook", "type": "announcement",
                    "date": date.isoformat(), "announcement": announcement,
                    "user": user, "channel": 'announcements'})

@robot.respond(r'(?i)make a (new )?notebook (entry|template) for (?P<date>\d+/\d+/\d+)')
def make_entry(res):
    date = dateutil.parser.parse(res.match.group('date'))
    logging.debug("date: %s", date)
    logging.debug("db dump: %s", res.robot.brain.db.all())
    announcements = res.robot.brain.db.search((where('plugin') == 'notebook')
        & (where('type') == 'announcement')
        & (where('date') == date.isoformat())
        )

    logging.debug("announcements are %s", announcements)

    if announcements != []:
        document = Document()
        document.add_page_break()
        document.add_heading('{date}, the BEC'.format(date=date.strftime('%m/%d/%Y')), level=1)
        document.add_paragraph('Present team members: <enter them here>')
        document.add_heading('Announcements:', level=2)

        users = []
        for announcement in announcements:
            if announcement['user'] not in users:
                users.append(announcement['user'])

        logging.debug("users are %s", users)

        for user in sorted(users):
            real_name = res.robot.slack_client.server.users.find(user).real_name
            logging.debug("announcing user %s is %s", user, real_name)
            document.add_paragraph("{}:".format(real_name))
            for announcement in announcements:
                if announcement['user'] == user:
                    document.add_paragraph("{}".format(announcement['announcement']),
                                           style='ListBullet')

        document.save('test.docx')
<|endoftext|>


<commit_before>import os
import traceback

import discord
from discord.ext import commands

from pony import orm

DESCRIPTION = """
Hi, I'm Yutu!
I'm the bot for the Velvet fan discord.
I'm still learning so sorry if I do something wrong.
You can ask my programmer @Harkonen if you want to know more about me.
"""

class Yutu(commands.Bot):
    def __init__(self):
        super().__init__(commands.when_mentioned_or("~"),
                         game=discord.Game(name="~help"),
                         description=DESCRIPTION,
                         pm_help=None)
        self.db = orm.Database()
        self.get_command('help').after_invoke(self.post_help)

    async def post_help(self, ctx: commands.Context):
        await ctx.message.add_reaction("✅")

    async def on_ready(self):
        print('We have logged in as {0.user}'.format(self))
        self.owner_id = (await self.application_info()).owner.id
        await self.pm_owner(content="Yutu starting up")

    async def pm_owner(self, *args, **kwargs):
        owner = self.get_user(self.owner_id)
        await owner.send(*args, **kwargs)

    async def on_command_error(self, ctx: commands.Context, exception: Exception):
        if(isinstance(exception, commands.errors.MissingRequiredArgument) or
           isinstance(exception, commands.errors.BadArgument)):
            #await ctx.print_help()
            await self.pm_owner(content="".join(traceback.format_exception(None, exception, None)))
        elif isinstance(exception, commands.CommandOnCooldown):
            await ctx.send(content=str(exception))
        elif isinstance(exception, commands.MissingPermissions):
            await ctx.send(content="I'm sorry {}, I can't let you do that.".format(ctx.author.mention))
        elif not isinstance(exception, commands.CommandNotFound):
            await self.pm_owner(content="".join(traceback.format_exception(None, exception, None)))
        await super().on_command_error(ctx, exception)
<commit_msg>Remove debugging text<commit_after>import os
import traceback

import discord
from discord.ext import commands

from pony import orm

DESCRIPTION = """
Hi, I'm Yutu!
I'm the bot for the Velvet fan discord.
I'm still learning so sorry if I do something wrong.
You can ask my programmer @Harkonen if you want to know more about me.
"""

class Yutu(commands.Bot):
    def __init__(self):
        super().__init__(commands.when_mentioned_or("~"),
                         game=discord.Game(name="~help"),
                         description=DESCRIPTION,
                         pm_help=None)
        self.db = orm.Database()
        self.get_command('help').after_invoke(self.post_help)

    async def post_help(self, ctx: commands.Context):
        await ctx.message.add_reaction("✅")

    async def on_ready(self):
        print('We have logged in as {0.user}'.format(self))
        self.owner_id = (await self.application_info()).owner.id
        await self.pm_owner(content="Yutu starting up")

    async def pm_owner(self, *args, **kwargs):
        owner = self.get_user(self.owner_id)
        await owner.send(*args, **kwargs)

    async def on_command_error(self, ctx: commands.Context, exception: Exception):
        if(isinstance(exception, commands.errors.MissingRequiredArgument) or
           isinstance(exception, commands.errors.BadArgument)):
            await ctx.print_help()
        elif isinstance(exception, commands.CommandOnCooldown):
            await ctx.send(content=str(exception))
        elif isinstance(exception, commands.MissingPermissions):
            await ctx.send(content="I'm sorry {}, I can't let you do that.".format(ctx.author.mention))
        elif not isinstance(exception, commands.CommandNotFound):
            await self.pm_owner(content="".join(traceback.format_exception(None, exception, None)))
        await super().on_command_error(ctx, exception)
<|endoftext|>