<commit_before># Copyright 2011-2014 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import unicode_literals

"""Utilities for using Motor with Tornado web applications."""

import datetime
import email.utils
import mimetypes
import time

import tornado.web
from tornado import gen

import gridfs
import motor

# TODO: this class is not a drop-in replacement for StaticFileHandler.
#   StaticFileHandler provides class method make_static_url, which appends
#   an MD5 of the static file's contents. Templates thus can do
#   {{ static_url('image.png') }} and get "/static/image.png?v=1234abcdef",
#   which is cached forever. Problem is, it calculates the MD5 synchronously.
#   Two options: keep a synchronous GridFS available to get each grid file's
#   MD5 synchronously for every static_url call, or find some other idiom.

class GridFSHandler(tornado.web.RequestHandler):
    """A handler that can serve content from `GridFS`_, very similar to
    :class:`tornado.web.StaticFileHandler`.

    .. code-block:: python

        db = motor.MotorClient().my_database
        application = web.Application([
            (r"/static/(.*)", web.GridFSHandler, {"database": db}),
        ])

    By default, requests' If-Modified-Since headers are honored, but no
    specific cache-control timeout is sent to clients. Thus each request for
    a GridFS file requires a quick check of the file's ``uploadDate`` in
    MongoDB. Override :meth:`get_cache_time` in a subclass to customize this.
    """
    def initialize(self, database, root_collection='fs'):
        self.database = database
        self.root_collection = root_collection

    def get_gridfs_file(self, fs, path):
        """Overridable method to choose a GridFS file to serve at a URL.

        By default, if a URL pattern like ``"/static/(.*)"`` is mapped to this
        ``GridFSHandler``, then the trailing portion of the URL is used as the
        filename, so a request for "/static/image.png" results in a call
        to :meth:`MotorGridFS.get` with "image.png" as the ``filename``
        argument. To customize the mapping of path to GridFS file, override
        ``get_gridfs_file`` and return a Future :class:`~motor.MotorGridOut`
        from it.

        For example, to retrieve the file by ``_id`` instead of filename::

            class CustomGridFSHandler(motor.web.GridFSHandler):
                def get_gridfs_file(self, fs, path):
                    # Path is interpreted as _id instead of name.
                    # Return a Future MotorGridOut.
                    return fs.get(file_id=ObjectId(path))

        :Parameters:
          - `fs`: An open :class:`~motor.MotorGridFS` object
          - `path`: A string, the trailing portion of the URL pattern being
            served

        .. versionchanged:: 0.2
           ``get_gridfs_file`` no longer accepts a callback, instead returns
           a Future.
        """
        return fs.get_last_version(path)  # A Future MotorGridOut

    @gen.coroutine
    def get(self, path, include_body=True):
        fs = motor.MotorGridFS(self.database, self.root_collection)

        try:
            gridout = yield self.get_gridfs_file(fs, path)
        except gridfs.NoFile:
            raise tornado.web.HTTPError(404)

        # If-Modified-Since header is only good to the second.
        modified = gridout.upload_date.replace(microsecond=0)
        self.set_header("Last-Modified", modified)

        # MD5 is calculated on the MongoDB server when GridFS file is created
        self.set_header("Etag", '"%s"' % gridout.md5)

        mime_type = gridout.content_type

        # If content type is not defined, try to check it with mimetypes
        if mime_type is None:
            mime_type, encoding = mimetypes.guess_type(path)

        # Starting from here, largely a copy of StaticFileHandler
        if mime_type:
            self.set_header("Content-Type", mime_type)

        cache_time = self.get_cache_time(path, modified, mime_type)

        if cache_time > 0:
            self.set_header("Expires", datetime.datetime.utcnow() +
                                       datetime.timedelta(seconds=cache_time))
            self.set_header("Cache-Control", "max-age=" + str(cache_time))
        else:
            self.set_header("Cache-Control", "public")

        self.set_extra_headers(path, gridout)

        # Check the If-Modified-Since, and don't send the result if the
        # content has not been modified
        ims_value = self.request.headers.get("If-Modified-Since")
        if ims_value is not None:
            date_tuple = email.utils.parsedate(ims_value)
            if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
            if if_since >= modified:
                self.set_status(304)
                return

        # Same for Etag
        etag = self.request.headers.get("If-None-Match")
        if etag is not None and etag.strip('"') == gridout.md5:
            self.set_status(304)
            return

        self.set_header("Content-Length", gridout.length)
        if include_body:
            yield gridout.stream_to_handler(self)

        # Needed until fix for Tornado bug 751 is released, see
        # https://github.com/facebook/tornado/issues/751 and
        # https://github.com/facebook/tornado/commit/5491685
        self.finish()

    def head(self, path):
        # get() is a coroutine. Return its Future.
        return self.get(path, include_body=False)

    def get_cache_time(self, path, modified, mime_type):
        """Override to customize cache control behavior.

        Return a positive number of seconds to trigger aggressive caching or 0
        to mark resource as cacheable, only. 0 is the default.
        """
        return 0

    def set_extra_headers(self, path, gridout):
        """For subclass to add extra headers to the response"""
        pass
<commit_msg>Remove some docstring markup that isn't working.<commit_after># Copyright 2011-2014 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import unicode_literals

"""Utilities for using Motor with Tornado web applications."""

import datetime
import email.utils
import mimetypes
import time

import tornado.web
from tornado import gen

import gridfs
import motor

# TODO: this class is not a drop-in replacement for StaticFileHandler.
#   StaticFileHandler provides class method make_static_url, which appends
#   an MD5 of the static file's contents. Templates thus can do
#   {{ static_url('image.png') }} and get "/static/image.png?v=1234abcdef",
#   which is cached forever. Problem is, it calculates the MD5 synchronously.
#   Two options: keep a synchronous GridFS available to get each grid file's
#   MD5 synchronously for every static_url call, or find some other idiom.

class GridFSHandler(tornado.web.RequestHandler):
    """A handler that can serve content from GridFS, very similar to
    :class:`tornado.web.StaticFileHandler`.

    .. code-block:: python

        db = motor.MotorClient().my_database
        application = web.Application([
            (r"/static/(.*)", web.GridFSHandler, {"database": db}),
        ])

    By default, requests' If-Modified-Since headers are honored, but no
    specific cache-control timeout is sent to clients. Thus each request for
    a GridFS file requires a quick check of the file's ``uploadDate`` in
    MongoDB. Override :meth:`get_cache_time` in a subclass to customize this.
    """
    def initialize(self, database, root_collection='fs'):
        self.database = database
        self.root_collection = root_collection

    def get_gridfs_file(self, fs, path):
        """Overridable method to choose a GridFS file to serve at a URL.

        By default, if a URL pattern like ``"/static/(.*)"`` is mapped to this
        ``GridFSHandler``, then the trailing portion of the URL is used as the
        filename, so a request for "/static/image.png" results in a call
        to :meth:`MotorGridFS.get` with "image.png" as the ``filename``
        argument. To customize the mapping of path to GridFS file, override
        ``get_gridfs_file`` and return a Future :class:`~motor.MotorGridOut`
        from it.

        For example, to retrieve the file by ``_id`` instead of filename::

            class CustomGridFSHandler(motor.web.GridFSHandler):
                def get_gridfs_file(self, fs, path):
                    # Path is interpreted as _id instead of name.
                    # Return a Future MotorGridOut.
                    return fs.get(file_id=ObjectId(path))

        :Parameters:
          - `fs`: An open :class:`~motor.MotorGridFS` object
          - `path`: A string, the trailing portion of the URL pattern being
            served

        .. versionchanged:: 0.2
           ``get_gridfs_file`` no longer accepts a callback, instead returns
           a Future.
        """
        return fs.get_last_version(path)  # A Future MotorGridOut

    @gen.coroutine
    def get(self, path, include_body=True):
        fs = motor.MotorGridFS(self.database, self.root_collection)

        try:
            gridout = yield self.get_gridfs_file(fs, path)
        except gridfs.NoFile:
            raise tornado.web.HTTPError(404)

        # If-Modified-Since header is only good to the second.
        modified = gridout.upload_date.replace(microsecond=0)
        self.set_header("Last-Modified", modified)

        # MD5 is calculated on the MongoDB server when GridFS file is created
        self.set_header("Etag", '"%s"' % gridout.md5)

        mime_type = gridout.content_type

        # If content type is not defined, try to check it with mimetypes
        if mime_type is None:
            mime_type, encoding = mimetypes.guess_type(path)

        # Starting from here, largely a copy of StaticFileHandler
        if mime_type:
            self.set_header("Content-Type", mime_type)

        cache_time = self.get_cache_time(path, modified, mime_type)

        if cache_time > 0:
            self.set_header("Expires", datetime.datetime.utcnow() +
                                       datetime.timedelta(seconds=cache_time))
            self.set_header("Cache-Control", "max-age=" + str(cache_time))
        else:
            self.set_header("Cache-Control", "public")

        self.set_extra_headers(path, gridout)

        # Check the If-Modified-Since, and don't send the result if the
        # content has not been modified
        ims_value = self.request.headers.get("If-Modified-Since")
        if ims_value is not None:
            date_tuple = email.utils.parsedate(ims_value)
            if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
            if if_since >= modified:
                self.set_status(304)
                return

        # Same for Etag
        etag = self.request.headers.get("If-None-Match")
        if etag is not None and etag.strip('"') == gridout.md5:
            self.set_status(304)
            return

        self.set_header("Content-Length", gridout.length)
        if include_body:
            yield gridout.stream_to_handler(self)

        # Needed until fix for Tornado bug 751 is released, see
        # https://github.com/facebook/tornado/issues/751 and
        # https://github.com/facebook/tornado/commit/5491685
        self.finish()

    def head(self, path):
        # get() is a coroutine. Return its Future.
        return self.get(path, include_body=False)

    def get_cache_time(self, path, modified, mime_type):
        """Override to customize cache control behavior.

        Return a positive number of seconds to trigger aggressive caching or 0
        to mark resource as cacheable, only. 0 is the default.
        """
        return 0

    def set_extra_headers(self, path, gridout):
        """For subclass to add extra headers to the response"""
        pass
<|endoftext|>


<commit_before>import arrow
import collections
from typing import Optional, Dict, Any, List, Mapping, Iterable

from ..exceptions import ShardIteratorExpired
from ..session import SessionWrapper
from ..util import Sentinel

# Approximate number of calls to fully traverse an empty shard
CALLS_TO_REACH_HEAD = 5

last_iterator = Sentinel("LastIterator")
missing = Sentinel("missing")

class Shard:
    def __init__(self, *, stream_arn: str, shard_id: str,
                 iterator_id: Optional[str]=None, iterator_type: Optional[str]=None,
                 sequence_number: Optional[str]=None, parent: Optional["Shard"]=None,
                 session: Optional[SessionWrapper]=None):
        # Set once on creation, never changes
        self.stream_arn = stream_arn
        # Set once on creation, never changes
        self.shard_id = shard_id
        # Changes frequently, not set initially
        # Iterators have a 15 minute lifetime, and need to be refreshed before then.
        # If they expire, a new one can be created deterministically IFF the Shard has
        # both a sequence_number and iterator_type of "at_sequence" or "after_sequence"
        # Iterators also change on nearly every GetRecords call.
        # When the end of a closed Shard is reached, this becomes None.
        self.iterator_id = iterator_id
        # Changes infrequently, not set initially
        # This will change on seek/jump, and the first time a record is encountered
        # (usually, from "trim_horizon" or "latest" to "at_sequence")
        self.iterator_type = iterator_type
        # Changes frequently, not set initially
        # See iterator_type
        self.sequence_number = sequence_number
        # Changes very infrequently, set initially
        # This will only change when the parent Shard previously existed, but has
        # since passed the 24 hour trim horizon.
        # After the initial set, this will never go from None -> Shard,
        # since Shards do not re-parent.
        self.parent = parent
        # Changes infrequently, set initially
        # Unlike parent, new children are added periodically (~4hrs) and as throughput
        # requires splitting a Shard.  A Shard will have exactly 0, 1, or 2 children.
        # A Shard with 0 children may be open, or there was a reduction in throughput.
        # A Shard with 1 child is closed, and did not split.
        # A Shard with 2 children is closed, due to an increase in throughput.  Updates to
        #   records that were in this Shard may be in either child Shard, but will consistently
        #   be written to the same child Shard (updates to record A will only go to one child, and not
        #   bounce back and forth).
        self.children = []
        # Changes infrequently, 0 initially
        # Tracks how many times a GetRecords call has returned no results, but a next iterator_id.
        # After ~5 empty responses, we can reasonably expect that the iterator is near the HEAD of an open Shard.
        # This dictates how hard we need to work to "catch up" a new iterator, in the face of empty results
        # (which provide no SequenceNumber or ApproximateCreationDateTime to approximate our location in the Stream).
        self.empty_responses = 0

        # Convenience so the Coordinator's session isn't explicitly needed for every call
        self.session = session

    def __repr__(self):
        if self.exhausted:
            # <Shard[exhausted, id='shardId-00000001414562045508-2bac9cd2']>
            details = "exhausted"
        elif self.iterator_type == "at_sequence":
            # <Shard[at=300000000000000499659, id='shardId-00000001414562045508-2bac9cd2']>
            details = "at_seq=" + repr(self.sequence_number)
        elif self.iterator_type == "after_sequence":
            # <Shard[after=300000000000000499659, id='shardId-00000001414562045508-2bac9cd2']>
            details = "after_seq=" + repr(self.sequence_number)
        elif self.iterator_type in ["trim_horizon", "latest"]:
            # <Shard[latest, id='shardId-00000001414562045508-2bac9cd2']>
            # <Shard[trim_horizon, id='shardId-00000001414562045508-2bac9cd2']>
            details = self.iterator_type
        else:
            # <Shard[id='shardId-00000001414562045508-2bac9cd2']>
            details = ""
        if details:
            details += ", "
        return "<{}[{}id={!r}]>".format(self.__class__.__name__, details, self.shard_id)

    def __next__(self) -> List[Dict[str, Any]]:
        try:
            return self.get_records()
        except ShardIteratorExpired:
            # Refreshing a sequence_number-based Shard iterator is deterministic;
            # if the iterator type is latest or trim_horizon, it's up to the caller to
            # decide how to proceed.
            if self.iterator_type in ["trim_horizon", "latest"]:
                raise

        # Since the expired iterator has a sequence_number, try to refresh automatically.
        # This could still raise RecordsExpired, if the desired position fell behind the
        # the trim_horizon since it expired.
        self.jump_to(iterator_type=self.iterator_type, sequence_number=self.sequence_number)

        # If it didn't expire, let's try returning records once more.
        return self.get_records()

    def __eq__(self, other):
        try:
            return (
                (self.token == other.token) and
                (self.iterator_id == other.iterator_id) and
                ({child.shard_id for child in self.children} == {child.shard_id for child in other.children})
            )
        except (AttributeError, TypeError):
            return False

    @property
    def exhausted(self) -> bool:
        return self.iterator_id is last_iterator

    @property
    def token(self) -> Dict[str, Any]:
        """Does not recursively tokenize children.

        Returns fields that may be redundant for generating a Stream token,
        such as stream_arn and shard_id.
        """
        # TODO logger.info when iterator_type is "trim_horizon" or "latest"
        token = {
            "stream_arn": self.stream_arn,
            "shard_id": self.shard_id,
            "iterator_type": self.iterator_type,
            "sequence_number": self.sequence_number,
        }
        if self.parent:
            token["parent"] = self.parent.shard_id
        return token

    def walk_tree(self) -> Iterable["Shard"]:
        """Generator that visits all shards in a shard tree"""
        shards = collections.deque([self])
        while shards:
            shard = shards.popleft()
            yield shard
            shards.extend(shard.children)

    def jump_to(self, *, iterator_type: str, sequence_number: Optional[str]=None) -> None:
        # Just a simple wrapper; let the caller handle RecordsExpired
        self.iterator_id = self.session.get_shard_iterator(
            stream_arn=self.stream_arn,
            shard_id=self.shard_id,
            iterator_type=iterator_type,
            sequence_number=sequence_number)
        self.iterator_type = iterator_type
        self.sequence_number = sequence_number
        self.empty_responses = 0

    def seek_to(self, position: arrow.Arrow) -> List[Dict[str, Any]]:
        """Move the Shard's iterator to the earliest record that after the given time.

        Returns the first records at or past ``position``.  If the list is empty,
        the seek failed to find records, either because the Shard is exhausted or it
        reached the HEAD of an open Shard.
        """
        # 0) We have no way to associate the date with a position, so we simply have to go through
        #    the entire Shard until we find a set of records with at least one with ApproxCreateDate >= position.
        self.jump_to(iterator_type="trim_horizon")

        # The timestamps from DynamoDBStreams are integers; truncate the
        # input position so we don't see 1476528055 < 1476528055.3 and
        # have an off-by-one index in the reverse enumeration.
        position = position.timestamp

        # Stop once the Shard is exhausted (can't possibly find the record)
        # or we've somewhat confidently caught the HEAD of an open Shard.
        while (not self.exhausted) and (self.empty_responses < CALLS_TO_REACH_HEAD):
            # Don't need to worry about RecordsExpired, since we're starting at trim_horizon.
            # Don't need to worry about ShardIteratorExpired, since we just created this one at trim_horizon.
            records = self.get_records()
            # Shortcut: we need AT LEAST one record to be after the position, so check the last record.
            # if it's before the position, all of the records in this response are.
            if records and records[-1]["meta"]["created_at"].timestamp >= position:
                # Reverse search is faster (on average; they're still both O(n) worst).
                # since we're looking for the first number *below* the position.
                for offset, record in enumerate(reversed(records)):
                    if record["meta"]["created_at"].timestamp < position:
                        index = len(records) - offset
                        return records[index:]
                # If the loop above finished, it means ALL the records are after the position.
                return records

        # Either exhausted the Shard, or caught up to HEAD.
        # It's only a failure to seek if the Shard is exhausted.
        return []

    def load_children(self) -> None:
        """Try to load the shard's children from DynamoDB if it doesn't have any.

        Loads all shards that have this shard as an ancestor.
        """
        # If a Shard has children, that number will never change.
        # Children are the result of exactly one event:
        #   increased throughput -> exactly 2 children
        #         open for ~4hrs -> at most 1 child
        if self.children:
            return self.children

        # ParentShardId -> [Shard, ...]
        by_parent = collections.defaultdict(list)
        # ShardId -> Shard
        by_id = {}
        for shard in self.session.describe_stream(
                stream_arn=self.stream_arn,
                first_shard=self.shard_id)["Shards"]:
            parent_list = by_parent[shard.get("ParentShardId")]
            shard = Shard(
                stream_arn=self.stream_arn,
                shard_id=shard["ShardId"],
                parent=shard.get("ParentShardId"),
                session=self.session)
            parent_list.append(shard)
            by_id[shard.shard_id] = shard

        # Find this shard when looking up shards by ParentShardId
        by_id[self.shard_id] = self

        # Insert this shard's children, then handle its child's descendants etc.
        to_insert = collections.deque(by_parent[self.shard_id])
        while to_insert:
            shard = to_insert.popleft()
            # ParentShardId -> Shard
            shard.parent = by_id[shard.parent]
            shard.parent.children.append(shard)
            # Continue for any shards that have this shard as their parent
            to_insert.extend(by_parent[shard.shard_id])

        return self.children

    def get_records(self) -> List[Dict[str, Any]]:
        """Call GetRecords and apply catch-up logic.  Updates shard.iterator_id.  No exception handling."""
        # Won't be able to find new records.
        if self.exhausted:
            return []

        # Already caught up, just the one call please.
        if self.empty_responses >= CALLS_TO_REACH_HEAD:
            return self._apply_get_records_response(self.session.get_stream_records(self.iterator_id))

        # Up to 5 calls to try and find a result
        while self.empty_responses < CALLS_TO_REACH_HEAD and not self.exhausted:
            records = self._apply_get_records_response(self.session.get_stream_records(self.iterator_id))
            if records:
                # Stop working the first time we find results.
                return records

        # Failed after 5 calls
        return []

    def _apply_get_records_response(self, response: Mapping[str, Any]) -> List[Dict[str, Any]]:
        records = response.get("Records", [])
        records = [reformat_record(record) for record in records]
        self.iterator_id = response.get("NextShardIterator", last_iterator)

        # The iterator state should ONLY be updated if there's no sequence_number already.
        # This ensures we can refresh from a fixed point, which is unnecessary if we have a number.
        # If sequence_number is set, we're risking data loss by skipping the existing checkpoint.
        if records and self.sequence_number is None:
            self.sequence_number = records[0]["meta"]["sequence_number"]
            self.iterator_type = "at_sequence"
        elif not records:
            self.empty_responses += 1
        return records

def reformat_record(record: Mapping[str, Any]) -> Dict[str, Any]:
    """Repack a record into a cleaner structure for consumption."""
    # Unwrap the inner structure, since most of it comes from here
    return {
        "key": record["dynamodb"].get("Keys", None),
        "new": record["dynamodb"].get("NewImage", None),
        "old": record["dynamodb"].get("OldImage", None),

        "meta": {
            "created_at": arrow.get(record["dynamodb"]["ApproximateCreationDateTime"]),
            "event": {
                "id": record["eventID"],
                "type": record["eventName"].lower(),
                "version": record["eventVersion"]
            },
            "sequence_number": record["dynamodb"]["SequenceNumber"],
        }
    }

def unpack_shards(shards: List[Mapping[str, Any]], stream_arn: str, session: SessionWrapper) -> Dict[str, Shard]:
    """List[Dict] -> Dict[shard_id, Shard].

    Each Shards' parent/children are hooked up with the other Shards in the list.
    """
    if not shards:
        return {}

    # When unpacking tokens, shard id key is "shard_id"
    # When unpacking DescribeStream responses, shard id key is "ShardId"
    if "ShardId" in shards[0]:
        shards = _translate_shards(shards)

    by_id = {shard_token["shard_id"]:
             Shard(stream_arn=stream_arn, shard_id=shard_token["shard_id"],
                   iterator_type=shard_token["iterator_type"], sequence_number=shard_token["sequence_number"],
                   parent=shard_token.get("parent"), session=session)
             for shard_token in shards}

    for shard in by_id.values():
        if shard.parent:
            shard.parent = by_id[shard.parent]
            shard.parent.children.append(shard)
    return by_id

def _translate_shards(shards: List[Mapping[str, Any]]) -> Iterable[Mapping[str, Any]]:
    """Converts the dicts from DescribeStream to the internal Shard format."""
    for shard in shards:
        yield {
            "shard_id": shard["ShardId"],
            "iterator_type": None,
            "sequence_number": None,
            "iterator_id": None,
            "parent": shard.get("ParentShardId")
        }
<commit_msg>Clean up Nones from Shard.token, relax token -> Shard loading #17<commit_after>import arrow
import collections
from typing import Optional, Dict, Any, List, Mapping, Iterable

from ..exceptions import ShardIteratorExpired
from ..session import SessionWrapper
from ..util import Sentinel

# Approximate number of calls to fully traverse an empty shard
CALLS_TO_REACH_HEAD = 5

last_iterator = Sentinel("LastIterator")
missing = Sentinel("missing")

class Shard:
    def __init__(self, *, stream_arn: str, shard_id: str,
                 iterator_id: Optional[str]=None, iterator_type: Optional[str]=None,
                 sequence_number: Optional[str]=None, parent: Optional["Shard"]=None,
                 session: Optional[SessionWrapper]=None):
        # Set once on creation, never changes
        self.stream_arn = stream_arn
        # Set once on creation, never changes
        self.shard_id = shard_id
        # Changes frequently, not set initially
        # Iterators have a 15 minute lifetime, and need to be refreshed before then.
        # If they expire, a new one can be created deterministically IFF the Shard has
        # both a sequence_number and iterator_type of "at_sequence" or "after_sequence"
        # Iterators also change on nearly every GetRecords call.
        # When the end of a closed Shard is reached, this becomes None.
        self.iterator_id = iterator_id
        # Changes infrequently, not set initially
        # This will change on seek/jump, and the first time a record is encountered
        # (usually, from "trim_horizon" or "latest" to "at_sequence")
        self.iterator_type = iterator_type
        # Changes frequently, not set initially
        # See iterator_type
        self.sequence_number = sequence_number
        # Changes very infrequently, set initially
        # This will only change when the parent Shard previously existed, but has
        # since passed the 24 hour trim horizon.
        # After the initial set, this will never go from None -> Shard,
        # since Shards do not re-parent.
        self.parent = parent
        # Changes infrequently, set initially
        # Unlike parent, new children are added periodically (~4hrs) and as throughput
        # requires splitting a Shard.  A Shard will have exactly 0, 1, or 2 children.
        # A Shard with 0 children may be open, or there was a reduction in throughput.
        # A Shard with 1 child is closed, and did not split.
        # A Shard with 2 children is closed, due to an increase in throughput.  Updates to
        #   records that were in this Shard may be in either child Shard, but will consistently
        #   be written to the same child Shard (updates to record A will only go to one child, and not
        #   bounce back and forth).
        self.children = []
        # Changes infrequently, 0 initially
        # Tracks how many times a GetRecords call has returned no results, but a next iterator_id.
        # After ~5 empty responses, we can reasonably expect that the iterator is near the HEAD of an open Shard.
        # This dictates how hard we need to work to "catch up" a new iterator, in the face of empty results
        # (which provide no SequenceNumber or ApproximateCreationDateTime to approximate our location in the Stream).
        self.empty_responses = 0

        # Convenience so the Coordinator's session isn't explicitly needed for every call
        self.session = session

    def __repr__(self):
        if self.exhausted:
            # <Shard[exhausted, id='shardId-00000001414562045508-2bac9cd2']>
            details = "exhausted"
        elif self.iterator_type == "at_sequence":
            # <Shard[at=300000000000000499659, id='shardId-00000001414562045508-2bac9cd2']>
            details = "at_seq=" + repr(self.sequence_number)
        elif self.iterator_type == "after_sequence":
            # <Shard[after=300000000000000499659, id='shardId-00000001414562045508-2bac9cd2']>
            details = "after_seq=" + repr(self.sequence_number)
        elif self.iterator_type in ["trim_horizon", "latest"]:
            # <Shard[latest, id='shardId-00000001414562045508-2bac9cd2']>
            # <Shard[trim_horizon, id='shardId-00000001414562045508-2bac9cd2']>
            details = self.iterator_type
        else:
            # <Shard[id='shardId-00000001414562045508-2bac9cd2']>
            details = ""
        if details:
            details += ", "
        return "<{}[{}id={!r}]>".format(self.__class__.__name__, details, self.shard_id)

    def __next__(self) -> List[Dict[str, Any]]:
        try:
            return self.get_records()
        except ShardIteratorExpired:
            # Refreshing a sequence_number-based Shard iterator is deterministic;
            # if the iterator type is latest or trim_horizon, it's up to the caller to
            # decide how to proceed.
            if self.iterator_type in ["trim_horizon", "latest"]:
                raise

        # Since the expired iterator has a sequence_number, try to refresh automatically.
        # This could still raise RecordsExpired, if the desired position fell behind the
        # the trim_horizon since it expired.
        self.jump_to(iterator_type=self.iterator_type, sequence_number=self.sequence_number)

        # If it didn't expire, let's try returning records once more.
        return self.get_records()

    def __eq__(self, other):
        try:
            return (
                (self.token == other.token) and
                (self.iterator_id == other.iterator_id) and
                ({child.shard_id for child in self.children} == {child.shard_id for child in other.children})
            )
        except (AttributeError, TypeError):
            return False

    @property
    def exhausted(self) -> bool:
        return self.iterator_id is last_iterator

    @property
    def token(self) -> Dict[str, Any]:
        """Does not recursively tokenize children.

        Returns fields that may be redundant for generating a Stream token,
        such as stream_arn and shard_id.
        """
        # TODO logger.info when iterator_type is "trim_horizon" or "latest"
        token = {
            "stream_arn": self.stream_arn,
            "shard_id": self.shard_id,
            "iterator_type": self.iterator_type,
            "sequence_number": self.sequence_number,
        }
        if self.parent:
            token["parent"] = self.parent.shard_id
        if not self.iterator_type:
            del token["iterator_type"]
        if not self.sequence_number:
            del token["sequence_number"]
        return token

    def walk_tree(self) -> Iterable["Shard"]:
        """Generator that visits all shards in a shard tree"""
        shards = collections.deque([self])
        while shards:
            shard = shards.popleft()
            yield shard
            shards.extend(shard.children)

    def jump_to(self, *, iterator_type: str, sequence_number: Optional[str]=None) -> None:
        # Just a simple wrapper; let the caller handle RecordsExpired
        self.iterator_id = self.session.get_shard_iterator(
            stream_arn=self.stream_arn,
            shard_id=self.shard_id,
            iterator_type=iterator_type,
            sequence_number=sequence_number)
        self.iterator_type = iterator_type
        self.sequence_number = sequence_number
        self.empty_responses = 0

    def seek_to(self, position: arrow.Arrow) -> List[Dict[str, Any]]:
        """Move the Shard's iterator to the earliest record that after the given time.

        Returns the first records at or past ``position``.  If the list is empty,
        the seek failed to find records, either because the Shard is exhausted or it
        reached the HEAD of an open Shard.
        """
        # 0) We have no way to associate the date with a position, so we simply have to go through
        #    the entire Shard until we find a set of records with at least one with ApproxCreateDate >= position.
        self.jump_to(iterator_type="trim_horizon")

        # The timestamps from DynamoDBStreams are integers; truncate the
        # input position so we don't see 1476528055 < 1476528055.3 and
        # have an off-by-one index in the reverse enumeration.
        position = position.timestamp

        # Stop once the Shard is exhausted (can't possibly find the record)
        # or we've somewhat confidently caught the HEAD of an open Shard.
        while (not self.exhausted) and (self.empty_responses < CALLS_TO_REACH_HEAD):
            # Don't need to worry about RecordsExpired, since we're starting at trim_horizon.
            # Don't need to worry about ShardIteratorExpired, since we just created this one at trim_horizon.
            records = self.get_records()
            # Shortcut: we need AT LEAST one record to be after the position, so check the last record.
            # if it's before the position, all of the records in this response are.
            if records and records[-1]["meta"]["created_at"].timestamp >= position:
                # Reverse search is faster (on average; they're still both O(n) worst).
                # since we're looking for the first number *below* the position.
                for offset, record in enumerate(reversed(records)):
                    if record["meta"]["created_at"].timestamp < position:
                        index = len(records) - offset
                        return records[index:]
                # If the loop above finished, it means ALL the records are after the position.
                return records

        # Either exhausted the Shard, or caught up to HEAD.
        # It's only a failure to seek if the Shard is exhausted.
        return []

    def load_children(self) -> None:
        """Try to load the shard's children from DynamoDB if it doesn't have any.

        Loads all shards that have this shard as an ancestor.
        """
        # If a Shard has children, that number will never change.
        # Children are the result of exactly one event:
        #   increased throughput -> exactly 2 children
        #         open for ~4hrs -> at most 1 child
        if self.children:
            return self.children

        # ParentShardId -> [Shard, ...]
        by_parent = collections.defaultdict(list)
        # ShardId -> Shard
        by_id = {}
        for shard in self.session.describe_stream(
                stream_arn=self.stream_arn,
                first_shard=self.shard_id)["Shards"]:
            parent_list = by_parent[shard.get("ParentShardId")]
            shard = Shard(
                stream_arn=self.stream_arn,
                shard_id=shard["ShardId"],
                parent=shard.get("ParentShardId"),
                session=self.session)
            parent_list.append(shard)
            by_id[shard.shard_id] = shard

        # Find this shard when looking up shards by ParentShardId
        by_id[self.shard_id] = self

        # Insert this shard's children, then handle its child's descendants etc.
        to_insert = collections.deque(by_parent[self.shard_id])
        while to_insert:
            shard = to_insert.popleft()
            # ParentShardId -> Shard
            shard.parent = by_id[shard.parent]
            shard.parent.children.append(shard)
            # Continue for any shards that have this shard as their parent
            to_insert.extend(by_parent[shard.shard_id])

        return self.children

    def get_records(self) -> List[Dict[str, Any]]:
        """Call GetRecords and apply catch-up logic.  Updates shard.iterator_id.  No exception handling."""
        # Won't be able to find new records.
        if self.exhausted:
            return []

        # Already caught up, just the one call please.
        if self.empty_responses >= CALLS_TO_REACH_HEAD:
            return self._apply_get_records_response(self.session.get_stream_records(self.iterator_id))

        # Up to 5 calls to try and find a result
        while self.empty_responses < CALLS_TO_REACH_HEAD and not self.exhausted:
            records = self._apply_get_records_response(self.session.get_stream_records(self.iterator_id))
            if records:
                # Stop working the first time we find results.
                return records

        # Failed after 5 calls
        return []

    def _apply_get_records_response(self, response: Mapping[str, Any]) -> List[Dict[str, Any]]:
        records = response.get("Records", [])
        records = [reformat_record(record) for record in records]
        self.iterator_id = response.get("NextShardIterator", last_iterator)

        # The iterator state should ONLY be updated if there's no sequence_number already.
        # This ensures we can refresh from a fixed point, which is unnecessary if we have a number.
        # If sequence_number is set, we're risking data loss by skipping the existing checkpoint.
        if records and self.sequence_number is None:
            self.sequence_number = records[0]["meta"]["sequence_number"]
            self.iterator_type = "at_sequence"
        elif not records:
            self.empty_responses += 1
        return records

def reformat_record(record: Mapping[str, Any]) -> Dict[str, Any]:
    """Repack a record into a cleaner structure for consumption."""
    # Unwrap the inner structure, since most of it comes from here
    return {
        "key": record["dynamodb"].get("Keys", None),
        "new": record["dynamodb"].get("NewImage", None),
        "old": record["dynamodb"].get("OldImage", None),

        "meta": {
            "created_at": arrow.get(record["dynamodb"]["ApproximateCreationDateTime"]),
            "event": {
                "id": record["eventID"],
                "type": record["eventName"].lower(),
                "version": record["eventVersion"]
            },
            "sequence_number": record["dynamodb"]["SequenceNumber"],
        }
    }

def unpack_shards(shards: List[Mapping[str, Any]], stream_arn: str, session: SessionWrapper) -> Dict[str, Shard]:
    """List[Dict] -> Dict[shard_id, Shard].

    Each Shards' parent/children are hooked up with the other Shards in the list.
    """
    if not shards:
        return {}

    # When unpacking tokens, shard id key is "shard_id"
    # When unpacking DescribeStream responses, shard id key is "ShardId"
    if "ShardId" in shards[0]:
        shards = _translate_shards(shards)

    by_id = {shard_token["shard_id"]:
             Shard(stream_arn=stream_arn, shard_id=shard_token["shard_id"],
                   iterator_type=shard_token.get("iterator_type"), sequence_number=shard_token.get("sequence_number"),
                   parent=shard_token.get("parent"), session=session)
             for shard_token in shards}

    for shard in by_id.values():
        if shard.parent:
            shard.parent = by_id[shard.parent]
            shard.parent.children.append(shard)
    return by_id

def _translate_shards(shards: List[Mapping[str, Any]]) -> Iterable[Mapping[str, Any]]:
    """Converts the dicts from DescribeStream to the internal Shard format."""
    for shard in shards:
        yield {
            "shard_id": shard["ShardId"],
            "iterator_type": None,
            "sequence_number": None,
            "iterator_id": None,
            "parent": shard.get("ParentShardId")
        }
<|endoftext|>


<commit_before>#Setup logging
import logging
log = logging.getLogger(__name__)

#Python Modules
from datetime import datetime

#Local Modules
from ..alarm import Alarm
from ..utils import *

#Exernal Modules
from twitter import Twitter, OAuth

class Twitter_Alarm(Alarm):

	#Gather settings and create alarm
	def __init__(self, settings):
		self.token = settings['access_token']
		self.token_key = settings['access_secret']
		self.con_secret = settings['consumer_key']
		self.con_secret_key = settings['consumer_secret']
		self.status = settings.get('status', "A wild <pkmn> has appeared! Available until <24h_time> (<time_left>). <gmaps>")
		self.connect()
		log.info("Twitter Alarm intialized.")
		self.client.statuses.update(status="%s: PokeAlarm has intialized!" % datetime.utcnow().strftime("%H:%M:%S"))

	#Establish connection with Twitter
	def connect(self):
		self.client = Twitter(
			auth=OAuth(self.token, self.token_key, self.con_secret, self.con_secret_key))

	#Post Pokemon Status
	def pokemon_alert(self, pkinfo):
		args = { "status": replace(self.status, pkinfo)}
		try_sending(log, self.connect, "Twitter", self.client.statuses.update, args)<commit_msg>[Enhancement] Add switch to disable first tweet at initialization (#140)<commit_after>#Setup logging
import logging
log = logging.getLogger(__name__)

#Python Modules
from datetime import datetime

#Local Modules
from ..alarm import Alarm
from ..utils import *

#Exernal Modules
from twitter import Twitter, OAuth

class Twitter_Alarm(Alarm):

	#Gather settings and create alarm
	def __init__(self, settings):
		self.token = settings['access_token']
		self.token_key = settings['access_secret']
		self.con_secret = settings['consumer_key']
		self.con_secret_key = settings['consumer_secret']
		self.status = settings.get('status', "A wild <pkmn> has appeared! Available until <24h_time> (<time_left>). <gmaps>")
		self.startup_message = settings.get('startup_message', "True")
		self.connect()
		log.info("Twitter Alarm intialized.")
		if parse_boolean(self.startup_message):
			self.client.statuses.update(status="%s: PokeAlarm has intialized!" % datetime.now().strftime("%H:%M:%S"))

	#Establish connection with Twitter
	def connect(self):
		self.client = Twitter(
			auth=OAuth(self.token, self.token_key, self.con_secret, self.con_secret_key))

	#Post Pokemon Status
	def pokemon_alert(self, pkinfo):
		args = { "status": replace(self.status, pkinfo)}
		try_sending(log, self.connect, "Twitter", self.client.statuses.update, args)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
import select
import socket
import ssl
import time
import traceback

import pycurl

from threading import Thread

from module.Api import PackageDoesNotExists, FileDoesNotExists
from module.plugins.internal.Notifier import Notifier
from module.plugins.internal.misc import format_size

class IRC(Thread, Notifier):
    __name__    = "IRC"
    __type__    = "hook"
    __version__ = "0.21"
    __status__  = "testing"

    __config__ = [("activated", "bool", "Activated"                                    , False                    ),
                  ("host"     , "str" , "IRC-Server Address"                           , "Enter your server here!"),
                  ("port"     , "int" , "IRC-Server Port"                              , 6667                     ),
                  ("ident"    , "str" , "Clients ident"                                , "pyload-irc"             ),
                  ("realname" , "str" , "Realname"                                     , "pyload-irc"             ),
                  ("ssl"      , "bool", "Use SSL"                                      , False                    ),
                  ("nick"     , "str" , "Nickname the Client will take"                , "pyLoad-IRC"             ),
                  ("owner"    , "str" , "Nickname the Client will accept commands from", "Enter your nick here!"  ),
                  ("info_file", "bool", "Inform about every file finished"             , False                    ),
                  ("info_pack", "bool", "Inform about every package finished"          , True                     ),
                  ("captcha"  , "bool", "Send captcha requests"                        , True                     )]

    __description__ = """Connect to irc and let owner perform different tasks"""
    __license__     = "GPLv3"
    __authors__     = [("Jeix", "Jeix@hasnomail.com")]

    def __init__(self, *args, **kwargs):
        Thread.__init__(self)
        Addon.__init__(self, *args, **kwargs)
        self.setDaemon(True)

    def activate(self):
        self.abort = False
        self.more = []
        self.new_package = {}

        self.start()

    def package_finished(self, pypack):
        try:
            if self.config.get('info_pack'):
                self.response(_("Package finished: %s") % pypack.name)

        except Exception:
            pass

    def download_finished(self, pyfile):
        try:
            if self.config.get('info_file'):
                self.response(
                    _("Download finished: %(name)s @ %(plugin)s ") % {'name': pyfile.name, 'plugin': pyfile.pluginname})

        except Exception:
            pass

    def captcha_task(self, task):
        if self.config.get('captcha') and task.isTextual():
            task.handler.append(self)
            task.setWaiting(60)

            html = self.load("http://www.freeimagehosting.net/upload.php",
                          post={'attached': (pycurl.FORM_FILE, task.captchaFile)})

            url = re.search(r'\[img\]([^\[]+)\[/img\]\[/url\]', html).group(1)
            self.response(_("New Captcha Request: %s") % url)
            self.response(_("Answer with 'c %s text on the captcha'") % task.id)

    def run(self):
        #: Connect to IRC etc.
        self.sock = socket.socket()
        host = self.config.get('host')
        self.sock.connect((host, self.config.get('port')))

        if self.config.get('ssl'):
            self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)  #@TODO: support certificate

        nick = self.config.get('nick')
        self.sock.send("NICK %s\r\n" % nick)
        self.sock.send("USER %s %s bla :%s\r\n" % (nick, host, nick))
        for t in self.config.get('owner').split():
            if t.strip().startswith("#"):
                self.sock.send("JOIN %s\r\n" % t.strip())
        self.log_info(_("Connected to"), host)
        self.log_info(_("Switching to listening mode!"))
        try:
            self.main_loop()

        except IRCError, ex:
            self.sock.send("QUIT :byebye\r\n")
            if self.pyload.debug:
                traceback.print_exc()
            self.sock.close()

    def main_loop(self):
        readbuffer = ""
        while True:
            time.sleep(1)
            fdset = select.select([self.sock], [], [], 0)
            if self.sock not in fdset[0]:
                continue

            if self.abort:
                raise IRCError("quit")

            readbuffer += self.sock.recv(1024)
            temp = readbuffer.split("\n")
            readbuffer = temp.pop()

            for line in temp:
                line = line.rstrip()
                first = line.split()

                if first[0] == "PING":
                    self.sock.send("PONG %s\r\n" % first[1])

                if first[0] == "ERROR":
                    raise IRCError(line)

                msg = line.split(None, 3)
                if len(msg) < 4:
                    continue

                msg = {
                    'origin': msg[0][1:],
                    'action': msg[1],
                    'target': msg[2],
                    'text': msg[3][1:]
                }

                self.handle_events(msg)

    def handle_events(self, msg):
        if not msg['origin'].split("!", 1)[0] in self.config.get('owner').split():
            return

        if msg['target'].split("!", 1)[0] != self.config.get('nick'):
            return

        if msg['action'] != "PRIVMSG":
            return

        #: HANDLE CTCP ANTI FLOOD/BOT PROTECTION
        if msg['text'] == "\x01VERSION\x01":
            self.log_debug("Sending CTCP VERSION")
            self.sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface"))
            return
        elif msg['text'] == "\x01TIME\x01":
            self.log_debug("Sending CTCP TIME")
            self.sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time()))
            return
        elif msg['text'] == "\x01LAG\x01":
            self.log_debug("Received CTCP LAG")  #: don't know how to answer
            return

        trigger = "pass"
        args = None

        try:
            temp = msg['text'].split()
            trigger = temp[0]
            if len(temp) > 1:
                args = temp[1:]

        except Exception:
            pass

        handler = getattr(self, "event_%s" % trigger, self.event_pass)
        try:
            res = handler(args)
            for line in res:
                self.response(line, msg['origin'])

        except Exception, e:
            self.log_error(e, trace=True)

    def response(self, msg, origin=""):
        if origin == "":
            for t in self.config.get('owner').split():
                self.sock.send("PRIVMSG %s :%s\r\n" % (t.strip(), msg))
        else:
            self.sock.send("PRIVMSG %s :%s\r\n" % (origin.split("!", 1)[0], msg))

        #### Events

    def event_pass(self, args):
        return []

    def event_status(self, args):
        downloads = self.pyload.api.statusDownloads()
        if not downloads:
            return ["INFO: There are no active downloads currently."]

        temp_progress = ""
        lines = ["ID - Name - Status - Speed - ETA - Progress"]
        for data in downloads:

            if data.status == 5:
                temp_progress = data.format_wait
            else:
                temp_progress = "%d%% (%s)" % (data.percent, data.format_size)

            lines.append("#%d - %s - %s - %s - %s - %s" %
                         (
                             data.fid,
                             data.name,
                             data.statusmsg,
                             "%s/s" % format_size(data.speed),
                             "%s" % data.format_eta,
                             temp_progress
                         ))
        return lines

    def event_queue(self, args):
        pdata = self.pyload.api.getQueueData()

        if not pdata:
            return ["INFO: There are no packages in queue."]

        lines = []
        for pack in pdata:
            lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links)))

        return lines

    def event_collector(self, args):
        pdata = self.pyload.api.getCollectorData()
        if not pdata:
            return ["INFO: No packages in collector!"]

        lines = []
        for pack in pdata:
            lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links)))

        return lines

    def event_info(self, args):
        if not args:
            return ["ERROR: Use info like this: info <id>"]

        info = None
        try:
            info = self.pyload.api.getFileData(int(args[0]))

        except FileDoesNotExists:
            return ["ERROR: Link doesn't exists."]

        return ['LINK #%s: %s (%s) [%s][%s]' % (info.fid, info.name, info.format_size, info.statusmsg, info.plugin)]

    def event_packinfo(self, args):
        if not args:
            return ["ERROR: Use packinfo like this: packinfo <id>"]

        lines = []
        pack = None
        try:
            pack = self.pyload.api.getPackageData(int(args[0]))

        except PackageDoesNotExists:
            return ["ERROR: Package doesn't exists."]

        id = args[0]

        self.more = []

        lines.append('PACKAGE #%s: "%s" with %d links' % (id, pack.name, len(pack.links)))
        for pyfile in pack.links:
            self.more.append('LINK #%s: %s (%s) [%s][%s]' % (pyfile.fid, pyfile.name, pyfile.format_size,
                                                             pyfile.statusmsg, pyfile.plugin))

        if len(self.more) < 6:
            lines.extend(self.more)
            self.more = []
        else:
            lines.extend(self.more[:6])
            self.more = self.more[6:]
            lines.append("%d more links do display." % len(self.more))

        return lines

    def event_more(self, args):
        if not self.more:
            return ["No more information to display."]

        lines = self.more[:6]
        self.more = self.more[6:]
        lines.append("%d more links do display." % len(self.more))

        return lines

    def event_start(self, args):
        self.pyload.api.unpauseServer()
        return ["INFO: Starting downloads."]

    def event_stop(self, args):
        self.pyload.api.pauseServer()
        return ["INFO: No new downloads will be started."]

    def event_add(self, args):
        if len(args) < 2:
            return ['ERROR: Add links like this: "add <packagename|id> links". ',
                    "This will add the link <link> to to the package <package> / the package with id <id>!"]

        pack = args[0].strip()
        links = [x.strip() for x in args[1:]]

        count_added = 0
        count_failed = 0
        try:
            id = int(pack)
            pack = self.pyload.api.getPackageData(id)
            if not pack:
                return ["ERROR: Package doesn't exists."]

            #@TODO: add links

            return ["INFO: Added %d links to Package %s [#%d]" % (len(links), pack['name'], id)]

        except Exception:
            #: Create new package
            id = self.pyload.api.addPackage(pack, links, 1)
            return ["INFO: Created new Package %s [#%d] with %d links." % (pack, id, len(links))]

    def event_del(self, args):
        if len(args) < 2:
            return ["ERROR: Use del command like this: del -p|-l <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"]

        if args[0] == "-p":
            ret = self.pyload.api.deletePackages(map(int, args[1:]))
            return ["INFO: Deleted %d packages!" % len(args[1:])]

        elif args[0] == "-l":
            ret = self.pyload.api.delLinks(map(int, args[1:]))
            return ["INFO: Deleted %d links!" % len(args[1:])]

        else:
            return ["ERROR: Use del command like this: del <-p|-l> <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"]

    def event_push(self, args):
        if not args:
            return ["ERROR: Push package to queue like this: push <package id>"]

        id = int(args[0])
        try:
            info = self.pyload.api.getPackageInfo(id)
        except PackageDoesNotExists:
            return ["ERROR: Package #%d does not exist." % id]

        self.pyload.api.pushToQueue(id)
        return ["INFO: Pushed package #%d to queue." % id]

    def event_pull(self, args):
        if not args:
            return ["ERROR: Pull package from queue like this: pull <package id>."]

        id = int(args[0])
        if not self.pyload.api.getPackageData(id):
            return ["ERROR: Package #%d does not exist." % id]

        self.pyload.api.pullFromQueue(id)
        return ["INFO: Pulled package #%d from queue to collector." % id]

    def event_c(self, args):
        """
        Captcha answer
        """
        if not args:
            return ["ERROR: Captcha ID missing."]

        task = self.pyload.captchaManager.getTaskByID(args[0])
        if not task:
            return ["ERROR: Captcha Task with ID %s does not exists." % args[0]]

        task.setResult(" ".join(args[1:]))
        return ["INFO: Result %s saved." % " ".join(args[1:])]

    def event_help(self, args):
        lines = ["The following commands are available:",
                 "add <package|packid> <links> [...] Adds link to package. (creates new package if it does not exist)",
                 "queue                       Shows all packages in the queue",
                 "collector                   Shows all packages in collector",
                 "del -p|-l <id> [...]        Deletes all packages|links with the ids specified",
                 "info <id>                   Shows info of the link with id <id>",
                 "packinfo <id>               Shows info of the package with id <id>",
                 "more                        Shows more info when the result was truncated",
                 "start                       Starts all downloads",
                 "stop                        Stops the download (but not abort active downloads)",
                 "push <id>                   Push package to queue",
                 "pull <id>                   Pull package from queue",
                 "status                      Show general download status",
                 "help                        Shows this help message"]
        return lines

class IRCError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)
<commit_msg>[IRC] Update<commit_after># -*- coding: utf-8 -*-

import re
import select
import socket
import ssl
import time
import traceback

import pycurl

from threading import Thread

from module.Api import PackageDoesNotExists, FileDoesNotExists
from module.plugins.internal.Notifier import Notifier
from module.plugins.internal.misc import format_size

class IRC(Thread, Notifier):
    __name__    = "IRC"
    __type__    = "hook"
    __version__ = "0.22"
    __status__  = "testing"

    __config__ = [("activated", "bool", "Activated"                                    , False                    ),
                  ("host"     , "str" , "IRC-Server Address"                           , "Enter your server here!"),
                  ("port"     , "int" , "IRC-Server Port"                              , 6667                     ),
                  ("ident"    , "str" , "Clients ident"                                , "pyload-irc"             ),
                  ("realname" , "str" , "Realname"                                     , "pyload-irc"             ),
                  ("ssl"      , "bool", "Use SSL"                                      , False                    ),
                  ("nick"     , "str" , "Nickname the Client will take"                , "pyLoad-IRC"             ),
                  ("owner"    , "str" , "Nickname the Client will accept commands from", "Enter your nick here!"  ),
                  ("info_file", "bool", "Inform about every file finished"             , False                    ),
                  ("info_pack", "bool", "Inform about every package finished"          , True                     ),
                  ("captcha"  , "bool", "Send captcha requests"                        , True                     )]

    __description__ = """Connect to irc and let owner perform different tasks"""
    __license__     = "GPLv3"
    __authors__     = [("Jeix", "Jeix@hasnomail.com")]

    def __init__(self, *args, **kwargs):
        Thread.__init__(self)
        Notifier.__init__(self, *args, **kwargs)
        self.setDaemon(True)

    def activate(self):
        self.abort = False
        self.more = []
        self.new_package = {}

        self.start()

    def package_finished(self, pypack):
        try:
            if self.config.get('info_pack'):
                self.response(_("Package finished: %s") % pypack.name)

        except Exception:
            pass

    def download_finished(self, pyfile):
        try:
            if self.config.get('info_file'):
                self.response(
                    _("Download finished: %(name)s @ %(plugin)s ") % {'name': pyfile.name, 'plugin': pyfile.pluginname})

        except Exception:
            pass

    def captcha_task(self, task):
        if self.config.get('captcha') and task.isTextual():
            task.handler.append(self)
            task.setWaiting(60)

            html = self.load("http://www.freeimagehosting.net/upload.php",
                          post={'attached': (pycurl.FORM_FILE, task.captchaFile)})

            url = re.search(r'\[img\]([^\[]+)\[/img\]\[/url\]', html).group(1)
            self.response(_("New Captcha Request: %s") % url)
            self.response(_("Answer with 'c %s text on the captcha'") % task.id)

    def run(self):
        #: Connect to IRC etc.
        self.sock = socket.socket()
        host = self.config.get('host')
        self.sock.connect((host, self.config.get('port')))

        if self.config.get('ssl'):
            self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)  #@TODO: support certificate

        nick = self.config.get('nick')
        self.sock.send("NICK %s\r\n" % nick)
        self.sock.send("USER %s %s bla :%s\r\n" % (nick, host, nick))
        for t in self.config.get('owner').split():
            if t.strip().startswith("#"):
                self.sock.send("JOIN %s\r\n" % t.strip())
        self.log_info(_("Connected to"), host)
        self.log_info(_("Switching to listening mode!"))
        try:
            self.main_loop()

        except IRCError, ex:
            self.sock.send("QUIT :byebye\r\n")
            if self.pyload.debug:
                traceback.print_exc()
            self.sock.close()

    def main_loop(self):
        readbuffer = ""
        while True:
            time.sleep(1)
            fdset = select.select([self.sock], [], [], 0)
            if self.sock not in fdset[0]:
                continue

            if self.abort:
                raise IRCError("quit")

            readbuffer += self.sock.recv(1024)
            temp = readbuffer.split("\n")
            readbuffer = temp.pop()

            for line in temp:
                line = line.rstrip()
                first = line.split()

                if first[0] == "PING":
                    self.sock.send("PONG %s\r\n" % first[1])

                if first[0] == "ERROR":
                    raise IRCError(line)

                msg = line.split(None, 3)
                if len(msg) < 4:
                    continue

                msg = {
                    'origin': msg[0][1:],
                    'action': msg[1],
                    'target': msg[2],
                    'text': msg[3][1:]
                }

                self.handle_events(msg)

    def handle_events(self, msg):
        if not msg['origin'].split("!", 1)[0] in self.config.get('owner').split():
            return

        if msg['target'].split("!", 1)[0] != self.config.get('nick'):
            return

        if msg['action'] != "PRIVMSG":
            return

        #: HANDLE CTCP ANTI FLOOD/BOT PROTECTION
        if msg['text'] == "\x01VERSION\x01":
            self.log_debug("Sending CTCP VERSION")
            self.sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface"))
            return
        elif msg['text'] == "\x01TIME\x01":
            self.log_debug("Sending CTCP TIME")
            self.sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time()))
            return
        elif msg['text'] == "\x01LAG\x01":
            self.log_debug("Received CTCP LAG")  #: don't know how to answer
            return

        trigger = "pass"
        args = None

        try:
            temp = msg['text'].split()
            trigger = temp[0]
            if len(temp) > 1:
                args = temp[1:]

        except Exception:
            pass

        handler = getattr(self, "event_%s" % trigger, self.event_pass)
        try:
            res = handler(args)
            for line in res:
                self.response(line, msg['origin'])

        except Exception, e:
            self.log_error(e, trace=True)

    def response(self, msg, origin=""):
        if origin == "":
            for t in self.config.get('owner').split():
                self.sock.send("PRIVMSG %s :%s\r\n" % (t.strip(), msg))
        else:
            self.sock.send("PRIVMSG %s :%s\r\n" % (origin.split("!", 1)[0], msg))

        #### Events

    def event_pass(self, args):
        return []

    def event_status(self, args):
        downloads = self.pyload.api.statusDownloads()
        if not downloads:
            return ["INFO: There are no active downloads currently."]

        temp_progress = ""
        lines = ["ID - Name - Status - Speed - ETA - Progress"]
        for data in downloads:

            if data.status == 5:
                temp_progress = data.format_wait
            else:
                temp_progress = "%d%% (%s)" % (data.percent, data.format_size)

            lines.append("#%d - %s - %s - %s - %s - %s" %
                         (
                             data.fid,
                             data.name,
                             data.statusmsg,
                             "%s/s" % format_size(data.speed),
                             "%s" % data.format_eta,
                             temp_progress
                         ))
        return lines

    def event_queue(self, args):
        pdata = self.pyload.api.getQueueData()

        if not pdata:
            return ["INFO: There are no packages in queue."]

        lines = []
        for pack in pdata:
            lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links)))

        return lines

    def event_collector(self, args):
        pdata = self.pyload.api.getCollectorData()
        if not pdata:
            return ["INFO: No packages in collector!"]

        lines = []
        for pack in pdata:
            lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links)))

        return lines

    def event_info(self, args):
        if not args:
            return ["ERROR: Use info like this: info <id>"]

        info = None
        try:
            info = self.pyload.api.getFileData(int(args[0]))

        except FileDoesNotExists:
            return ["ERROR: Link doesn't exists."]

        return ['LINK #%s: %s (%s) [%s][%s]' % (info.fid, info.name, info.format_size, info.statusmsg, info.plugin)]

    def event_packinfo(self, args):
        if not args:
            return ["ERROR: Use packinfo like this: packinfo <id>"]

        lines = []
        pack = None
        try:
            pack = self.pyload.api.getPackageData(int(args[0]))

        except PackageDoesNotExists:
            return ["ERROR: Package doesn't exists."]

        id = args[0]

        self.more = []

        lines.append('PACKAGE #%s: "%s" with %d links' % (id, pack.name, len(pack.links)))
        for pyfile in pack.links:
            self.more.append('LINK #%s: %s (%s) [%s][%s]' % (pyfile.fid, pyfile.name, pyfile.format_size,
                                                             pyfile.statusmsg, pyfile.plugin))

        if len(self.more) < 6:
            lines.extend(self.more)
            self.more = []
        else:
            lines.extend(self.more[:6])
            self.more = self.more[6:]
            lines.append("%d more links do display." % len(self.more))

        return lines

    def event_more(self, args):
        if not self.more:
            return ["No more information to display."]

        lines = self.more[:6]
        self.more = self.more[6:]
        lines.append("%d more links do display." % len(self.more))

        return lines

    def event_start(self, args):
        self.pyload.api.unpauseServer()
        return ["INFO: Starting downloads."]

    def event_stop(self, args):
        self.pyload.api.pauseServer()
        return ["INFO: No new downloads will be started."]

    def event_add(self, args):
        if len(args) < 2:
            return ['ERROR: Add links like this: "add <packagename|id> links". ',
                    "This will add the link <link> to to the package <package> / the package with id <id>!"]

        pack = args[0].strip()
        links = [x.strip() for x in args[1:]]

        count_added = 0
        count_failed = 0
        try:
            id = int(pack)
            pack = self.pyload.api.getPackageData(id)
            if not pack:
                return ["ERROR: Package doesn't exists."]

            #@TODO: add links

            return ["INFO: Added %d links to Package %s [#%d]" % (len(links), pack['name'], id)]

        except Exception:
            #: Create new package
            id = self.pyload.api.addPackage(pack, links, 1)
            return ["INFO: Created new Package %s [#%d] with %d links." % (pack, id, len(links))]

    def event_del(self, args):
        if len(args) < 2:
            return ["ERROR: Use del command like this: del -p|-l <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"]

        if args[0] == "-p":
            ret = self.pyload.api.deletePackages(map(int, args[1:]))
            return ["INFO: Deleted %d packages!" % len(args[1:])]

        elif args[0] == "-l":
            ret = self.pyload.api.delLinks(map(int, args[1:]))
            return ["INFO: Deleted %d links!" % len(args[1:])]

        else:
            return ["ERROR: Use del command like this: del <-p|-l> <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"]

    def event_push(self, args):
        if not args:
            return ["ERROR: Push package to queue like this: push <package id>"]

        id = int(args[0])
        try:
            info = self.pyload.api.getPackageInfo(id)
        except PackageDoesNotExists:
            return ["ERROR: Package #%d does not exist." % id]

        self.pyload.api.pushToQueue(id)
        return ["INFO: Pushed package #%d to queue." % id]

    def event_pull(self, args):
        if not args:
            return ["ERROR: Pull package from queue like this: pull <package id>."]

        id = int(args[0])
        if not self.pyload.api.getPackageData(id):
            return ["ERROR: Package #%d does not exist." % id]

        self.pyload.api.pullFromQueue(id)
        return ["INFO: Pulled package #%d from queue to collector." % id]

    def event_c(self, args):
        """
        Captcha answer
        """
        if not args:
            return ["ERROR: Captcha ID missing."]

        task = self.pyload.captchaManager.getTaskByID(args[0])
        if not task:
            return ["ERROR: Captcha Task with ID %s does not exists." % args[0]]

        task.setResult(" ".join(args[1:]))
        return ["INFO: Result %s saved." % " ".join(args[1:])]

    def event_help(self, args):
        lines = ["The following commands are available:",
                 "add <package|packid> <links> [...] Adds link to package. (creates new package if it does not exist)",
                 "queue                       Shows all packages in the queue",
                 "collector                   Shows all packages in collector",
                 "del -p|-l <id> [...]        Deletes all packages|links with the ids specified",
                 "info <id>                   Shows info of the link with id <id>",
                 "packinfo <id>               Shows info of the package with id <id>",
                 "more                        Shows more info when the result was truncated",
                 "start                       Starts all downloads",
                 "stop                        Stops the download (but not abort active downloads)",
                 "push <id>                   Push package to queue",
                 "pull <id>                   Pull package from queue",
                 "status                      Show general download status",
                 "help                        Shows this help message"]
        return lines

class IRCError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: UTF-8

# This file is part of Addison Arches.
#
# Addison Arches is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Addison Arches is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Addison Arches.  If not, see <http://www.gnu.org/licenses/>.

import collections
import datetime
import enum
import itertools

from turberfield.dialogue.model import SceneScript
from turberfield.dialogue.types import DataObject
from turberfield.dialogue.types import EnumFactory
from turberfield.dialogue.types import Persona
from turberfield.dialogue.types import Player
from turberfield.dialogue.types import Stateful
from turberfield.utils.misc import Singleton

from bluemonday78 import __version__ as version
from bluemonday78.test.paths import GoldenPath

class Attitude(EnumFactory, enum.Enum):
    neutral = 0
    grumpy = 1
    affable = 2

class Spot(EnumFactory, enum.Enum):
    w12_ducane_prison = "gcpv4d"
    w12_ducane_prison_visiting = "gcpv4d252v5y"
    w12_ducane_prison_release = "gcpv4d1qmdzb"
    w12_goldhawk_cafe = "gcpufzg2512x"
    w12_goldhawk_tavern = "gcpufzbd8x5d"
    w12_latimer_arches = "gcpv4cxb3dh4"

class Via(EnumFactory, enum.Enum):
    block = 0
    forwd = 1
    bckwd = 2
    bidir = 3

class Narrator(Stateful): pass
class PrisonOfficer(Stateful, Persona): pass
class Prisoner(Stateful, Persona): pass
class PrisonVisitor(Stateful, Persona): pass
class Barman(Stateful, Persona): pass
class Hipster(Stateful, Persona): pass
class Character(Stateful, Persona): pass
class Location(Stateful, DataObject): pass

blue_monday = datetime.date(1978, 1, 16)

class Associations(Singleton):

    @classmethod
    def clear(cls):
        pass

    @classmethod
    def register(cls, rel, *args):
        cls().lookup

    def __init__(self):
        self.lookup = collections.OrderedDict([])

    def ensemble(query, limit=1):
        return collections.OrderedDict([(
            getattr(i, "_name", getattr(i, "label", type(i).__name__)).lower(),
            i.set_state(int(blue_monday.strftime("%Y%m%d"))))
            for i in (
                Narrator().set_state(Spot.w12_ducane_prison_visiting),
                Player(name="Mr Likely Story").set_state(Spot.w12_ducane_prison),
                Barman(
                    name="Mr Barry Latimer"
                ).set_state(
                    Attitude.neutral
                ).set_state(
                    Spot.w12_goldhawk_tavern
                ),
                Hipster(name="Mr Justin Cornelis Delcroix").set_state(
                    Spot.w12_goldhawk_tavern),
                PrisonOfficer(name="Mr Ray Farington").set_state(Spot.w12_ducane_prison_visiting),
                Prisoner(name="Mr Martin Sheppey").set_state(Spot.w12_ducane_prison),
                PrisonVisitor(name="Mrs Karen Sheppey").set_state(Spot.w12_ducane_prison),
                Character(name="Mr Ian Thomas").set_state(Spot.w12_goldhawk_tavern),
                Character(name="Mr Mike Phillips").set_state(Spot.w12_goldhawk_tavern),
                Character(name="Mr Matthew Waladli").set_state(Spot.w12_goldhawk_tavern),
                Location(label="Addison Arches 18A").set_state(Spot.w12_latimer_arches),
                Location(label="Wormwood Scrubs").set_state(Spot.w12_ducane_prison),
            )
        ])

def ensemble():
    return [
        i.set_state(int(blue_monday.strftime("%Y%m%d")))
        for i in (
            Narrator().set_state(Spot.w12_ducane_prison_visiting),
            Player(name="Mr Likely Story").set_state(Spot.w12_ducane_prison),
            Barman(
                name="Mr Barry Latimer"
            ).set_state(
                Attitude.neutral
            ).set_state(
                Spot.w12_goldhawk_tavern
            ),
            Hipster(name="Mr Justin Cornelis Delcroix").set_state(
                Spot.w12_goldhawk_tavern),
            PrisonOfficer(name="Mr Ray Farington").set_state(Spot.w12_ducane_prison_visiting),
            Prisoner(name="Mr Martin Sheppey").set_state(Spot.w12_ducane_prison),
            PrisonVisitor(name="Mrs Karen Sheppey").set_state(Spot.w12_ducane_prison),
            Character(name="Mr Ian Thomas").set_state(Spot.w12_goldhawk_tavern),
            Character(name="Mr Mike Phillips").set_state(Spot.w12_goldhawk_tavern),
            Character(name="Mr Matthew Waladli").set_state(Spot.w12_goldhawk_tavern),
            Location(label="Addison Arches 18A").set_state(Spot.w12_latimer_arches),
            Location(label="Wormwood Scrubs").set_state(Spot.w12_ducane_prison),
        )
    ]

references = ensemble() + [Attitude, Spot, Via]

local = SceneScript.Folder(
    pkg="bluemonday78",
    description="Location-specific elaboration.",
    metadata=[blue_monday],
    paths=[
        "w12_19780116_local/w12_latimer_arches_19780116.rst",
        "w12_19780116_local/w12_latimer_arches_19780117.rst",
    ],
    interludes=itertools.repeat(None)
)

justin = SceneScript.Folder(
    pkg="bluemonday78",
    description="Justin Delcroix has just got the sack.",
    metadata=[blue_monday],
    paths=[
        "justin_19780116_fired/sorrows.rst",
        "justin_19780116_fired/anguish.rst",
        "justin_19780116_fired/desparation.rst"
    ],
    interludes=itertools.repeat(None)
)

ray = SceneScript.Folder(
    pkg="bluemonday78",
    description="It's Ray Farington's last day.",
    metadata=[blue_monday],
    paths=[
        "ray_19780116_retires/admin.rst",
        "ray_19780116_retires/escape.rst",
        "ray_19780116_retires/homecoming.rst",
    ],
    interludes=itertools.repeat(None)
)

fade_in = SceneScript.Folder(
    pkg="bluemonday78",
    description="It's Ray Farington's last day.",
    metadata=[blue_monday],
    paths=[
        "ray_19780116_retires/step_forward.rst",
    ],
    interludes=itertools.repeat(GoldenPath.listen_to_karen)
)
plotlines = (justin, ray)
schedule = collections.deque([local])
schedule.extendleft(plotlines)
<commit_msg>Decide that classmethods construct an observer object first.<commit_after>#!/usr/bin/env python
# encoding: UTF-8

# This file is part of Addison Arches.
#
# Addison Arches is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Addison Arches is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Addison Arches.  If not, see <http://www.gnu.org/licenses/>.

import collections
import datetime
import enum
import itertools

from turberfield.dialogue.model import SceneScript
from turberfield.dialogue.types import DataObject
from turberfield.dialogue.types import EnumFactory
from turberfield.dialogue.types import Persona
from turberfield.dialogue.types import Player
from turberfield.dialogue.types import Stateful
from turberfield.utils.misc import Singleton

from bluemonday78 import __version__ as version
from bluemonday78.test.paths import GoldenPath

class Attitude(EnumFactory, enum.Enum):
    neutral = 0
    grumpy = 1
    affable = 2

class Spot(EnumFactory, enum.Enum):
    w12_ducane_prison = "gcpv4d"
    w12_ducane_prison_visiting = "gcpv4d252v5y"
    w12_ducane_prison_release = "gcpv4d1qmdzb"
    w12_goldhawk_cafe = "gcpufzg2512x"
    w12_goldhawk_tavern = "gcpufzbd8x5d"
    w12_latimer_arches = "gcpv4cxb3dh4"

class Via(EnumFactory, enum.Enum):
    block = 0
    forwd = 1
    bckwd = 2
    bidir = 3

class Narrator(Stateful): pass
class PrisonOfficer(Stateful, Persona): pass
class Prisoner(Stateful, Persona): pass
class PrisonVisitor(Stateful, Persona): pass
class Barman(Stateful, Persona): pass
class Hipster(Stateful, Persona): pass
class Character(Stateful, Persona): pass
class Location(Stateful, DataObject): pass

blue_monday = datetime.date(1978, 1, 16)

class Associations(Singleton):

    @classmethod
    def clear(cls):
        obj = cls()

    @classmethod
    def register(cls, rel, *args):
        obj = cls()

    def __init__(self):
        self.lookup = collections.OrderedDict([])

    def ensemble(query, limit=1):
        return collections.OrderedDict([(
            getattr(i, "_name", getattr(i, "label", type(i).__name__)).lower(),
            i.set_state(int(blue_monday.strftime("%Y%m%d"))))
            for i in (
                Narrator().set_state(Spot.w12_ducane_prison_visiting),
                Player(name="Mr Likely Story").set_state(Spot.w12_ducane_prison),
                Barman(
                    name="Mr Barry Latimer"
                ).set_state(
                    Attitude.neutral
                ).set_state(
                    Spot.w12_goldhawk_tavern
                ),
                Hipster(name="Mr Justin Cornelis Delcroix").set_state(
                    Spot.w12_goldhawk_tavern),
                PrisonOfficer(name="Mr Ray Farington").set_state(Spot.w12_ducane_prison_visiting),
                Prisoner(name="Mr Martin Sheppey").set_state(Spot.w12_ducane_prison),
                PrisonVisitor(name="Mrs Karen Sheppey").set_state(Spot.w12_ducane_prison),
                Character(name="Mr Ian Thomas").set_state(Spot.w12_goldhawk_tavern),
                Character(name="Mr Mike Phillips").set_state(Spot.w12_goldhawk_tavern),
                Character(name="Mr Matthew Waladli").set_state(Spot.w12_goldhawk_tavern),
                Location(label="Addison Arches 18A").set_state(Spot.w12_latimer_arches),
                Location(label="Wormwood Scrubs").set_state(Spot.w12_ducane_prison),
            )
        ])

def ensemble():
    return [
        i.set_state(int(blue_monday.strftime("%Y%m%d")))
        for i in (
            Narrator().set_state(Spot.w12_ducane_prison_visiting),
            Player(name="Mr Likely Story").set_state(Spot.w12_ducane_prison),
            Barman(
                name="Mr Barry Latimer"
            ).set_state(
                Attitude.neutral
            ).set_state(
                Spot.w12_goldhawk_tavern
            ),
            Hipster(name="Mr Justin Cornelis Delcroix").set_state(
                Spot.w12_goldhawk_tavern),
            PrisonOfficer(name="Mr Ray Farington").set_state(Spot.w12_ducane_prison_visiting),
            Prisoner(name="Mr Martin Sheppey").set_state(Spot.w12_ducane_prison),
            PrisonVisitor(name="Mrs Karen Sheppey").set_state(Spot.w12_ducane_prison),
            Character(name="Mr Ian Thomas").set_state(Spot.w12_goldhawk_tavern),
            Character(name="Mr Mike Phillips").set_state(Spot.w12_goldhawk_tavern),
            Character(name="Mr Matthew Waladli").set_state(Spot.w12_goldhawk_tavern),
            Location(label="Addison Arches 18A").set_state(Spot.w12_latimer_arches),
            Location(label="Wormwood Scrubs").set_state(Spot.w12_ducane_prison),
        )
    ]

references = ensemble() + [Attitude, Spot, Via]

local = SceneScript.Folder(
    pkg="bluemonday78",
    description="Location-specific elaboration.",
    metadata=[blue_monday],
    paths=[
        "w12_19780116_local/w12_latimer_arches_19780116.rst",
        "w12_19780116_local/w12_latimer_arches_19780117.rst",
    ],
    interludes=itertools.repeat(None)
)

justin = SceneScript.Folder(
    pkg="bluemonday78",
    description="Justin Delcroix has just got the sack.",
    metadata=[blue_monday],
    paths=[
        "justin_19780116_fired/sorrows.rst",
        "justin_19780116_fired/anguish.rst",
        "justin_19780116_fired/desparation.rst"
    ],
    interludes=itertools.repeat(None)
)

ray = SceneScript.Folder(
    pkg="bluemonday78",
    description="It's Ray Farington's last day.",
    metadata=[blue_monday],
    paths=[
        "ray_19780116_retires/admin.rst",
        "ray_19780116_retires/escape.rst",
        "ray_19780116_retires/homecoming.rst",
    ],
    interludes=itertools.repeat(None)
)

fade_in = SceneScript.Folder(
    pkg="bluemonday78",
    description="It's Ray Farington's last day.",
    metadata=[blue_monday],
    paths=[
        "ray_19780116_retires/step_forward.rst",
    ],
    interludes=itertools.repeat(GoldenPath.listen_to_karen)
)
plotlines = (justin, ray)
schedule = collections.deque([local])
schedule.extendleft(plotlines)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def square_matrix_multiply(A, B):
    """Square matrix multiplication by naive algorithm."""
    n = len(A)
    C = [[0 for j in range(n)] for i in range(n)]
    for i in range(n):
        for j in range(n):
            for k in range(n):
                C[i][j] += A[i][k] * B[k][j]
    return C

def _matrix_sum(A, B):
    m, n = len(A), len(A[0])
    C = [[0 for j in range(n)] for i in range(m)]
    for i in range(n):
        for j in range(n):
            C[i][j] = A[i][j] + B[i][j]
    return C

def square_matrix_multiply_dc(A, B):
    """Square matrix multiplication by simple divide & conquer algorithm"""
    n = len(A)
    print('n: {}'.format(n))
    C = [[0 for j in range(n)] for i in range(n)]

    if n == 1:
        C[0][0] = A[0][0] * B[0][0]
    else:
        # TODO: Come up with how to do index calculation.
        # C11 = A11 * B11 + A12 * B21
        C[:(n // 2)][:(n // 2)] = _square_matrix_sum(
            square_matrix_multiply_dc(
                A[:(n // 2)][:(n // 2)], B[:(n // 2)][:(n // 2)]),
            square_matrix_multiply_dc(
                A[:(n // 2)][(n // 2):], B[(n // 2):][:(n // 2)]))

        # C12 = A11 * B12 + A12 * B22
        C[:(n // 2)][(n // 2):] = _square_matrix_sum(
            square_matrix_multiply_dc(
                A[:(n // 2)][:(n // 2)], B[:(n // 2)][(n // 2):]),
            square_matrix_multiply_dc(
                A[:(n // 2)][(n // 2):], B[:(n // 2)][:(n // 2)]))

        # C21 = A21 * B11 + A22 * B21
        C[(n // 2):][:(n // 2)] = _square_matrix_sum(
            square_matrix_multiply_dc(
                A[(n // 2):][:(n // 2)], B[:(n // 2)][:(n // 2)]),
            square_matrix_multiply_dc(
                A[(n // 2):][(n // 2):], B[(n // 2):][:(n // 2)]))

        # C22 = A21 * B12 + A22 * B22
        C[(n // 2):][(n // 2):] = _square_matrix_sum(
            square_matrix_multiply_dc(
                A[(n // 2):][:(n // 2)], B[:(n // 2)][(n // 2):]),
            square_matrix_multiply_dc(
                A[(n // 2):][(n // 2):], B[(n // 2):][(n // 2):]))

    return C

def strassen_square_matrix_multiply(A, B):
    """square matrix multiplication by Strassen's algorithm.

    Note: Here assume the square matrix's dimension n is an exact power of 2.

    Time complexity: O(n^log7)
    Space complexity: O(1).
    """
    pass

def main():
    import time

    # From CLRS, Ex 4.2-1, p. 83.
    A = [[1, 3], [7, 5]]
    B = [[6, 8], [4, 2]]

    start_time = time.time()
    print('By naive algorithm:\n{}'.format(square_matrix_multiply(A, B)))
    print('Time: {}'.format(time.time() - start_time))

    # start_time = time.time()
    # print('By simple divide & conquer algorithm:\n{}'
    #       .format(square_matrix_multiply_dc(A, B)))
    # print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<commit_msg>Complete square_matrix_multiply_dc()<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def square_matrix_multiply(A, B):
    """Square matrix multiplication by naive algorithm."""
    n = len(A)
    C = [[0 for j in range(n)] for i in range(n)]
    for i in range(n):
        for j in range(n):
            for k in range(n):
                C[i][j] += A[i][k] * B[k][j]
    return C

def _matrix_index_calculation(A, row_indices, col_indices):
    """Sub-matrix by index calculation."""
    return [[A[i][j] for j in col_indices] for i in row_indices]

def _matrix_sum(A, B):
    """Sum of two matrices."""
    m, n = len(A), len(A[0])
    C = [[0 for j in range(n)] for i in range(m)]
    for i in range(n):
        for j in range(n):
            C[i][j] = A[i][j] + B[i][j]
    return C

def _matrix_assign(A, A_row_indices, A_col_indices, B):
    """Assign matrix B to sub-matrix A by index calculation."""
    for i in range(len(A_row_indices)):
        for j in range(len(A_col_indices)):
            A[A_row_indices[i]][A_col_indices[j]] = B[i][j]

def square_matrix_multiply_dc(A, B):
    """Square matrix multiplication by simple divide & conquer algorithm.

    Assume: the square matrix's dimension n is an exact power of 2.
    """
    n = len(A)
    C = [[0 for j in range(n)] for i in range(n)]

    if n == 1:
        C[0][0] = A[0][0] * B[0][0]
    else:
        # C11 = A11 * B11 + A12 * B21
        C11 = _matrix_sum(
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2), range(n // 2)),
                _matrix_index_calculation(B, range(n // 2), range(n // 2))
                ),
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2), range(n // 2, n)),
                _matrix_index_calculation(B, range(n // 2, n), range(n // 2))
                )
            )
        _matrix_assign(C, range(n // 2), range(n // 2), C11)

        # C12 = A11 * B12 + A12 * B22
        C12 = _matrix_sum(
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2), range(n // 2)),
                _matrix_index_calculation(B, range(n // 2), range(n // 2, n))
                ),
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2), range(n // 2, n)),
                _matrix_index_calculation(B, range(n // 2, n), range(n // 2, n))
                )
            )
        _matrix_assign(C, range(n // 2), range(n // 2, n), C12)

        # C21 = A21 * B11 + A22 * B21
        C21 = _matrix_sum(
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2, n), range(n // 2)),
                _matrix_index_calculation(B, range(n // 2), range(n // 2))
                ),
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2, n), range(n // 2, n)),
                _matrix_index_calculation(B, range(n // 2, n), range(n // 2))
                )
            )
        _matrix_assign(C, range(n // 2, n), range(n // 2), C21)

        # C22 = A21 * B12 + A22 * B22
        C22 = _matrix_sum(
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2, n), range(n // 2)),
                _matrix_index_calculation(B, range(n // 2), range(n // 2, n))
                ),
            square_matrix_multiply_dc(
                _matrix_index_calculation(A, range(n // 2, n), range(n // 2, n)),
                _matrix_index_calculation(B, range(n // 2, n), range(n // 2, n))
                )
            )
        _matrix_assign(C, range(n // 2, n), range(n // 2, n), C22)

    return C

def strassen_square_matrix_multiply(A, B):
    """square matrix multiplication by Strassen's algorithm.

    Assume: the square matrix's dimension n is an exact power of 2.

    Time complexity: O(n^log7)
    Space complexity: O(1).
    """
    pass

def main():
    import time

    # From CLRS, Ex 4.2-1, p. 83.
    A = [[1, 3], [7, 5]]
    B = [[6, 8], [4, 2]]
    # A = [[1, 3, 4, 6], [7, 5, 9, 5], [2, 6, 5, 7], [3, 7, 5, 4]]
    # B = [[6, 8, 5, 9], [4, 2, 8, 2], [3, 1, 9, 5], [5, 2, 1, 8]]

    start_time = time.time()
    print('By naive algorithm:\n{}'.format(square_matrix_multiply(A, B)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By simple divide & conquer algorithm:\n{}'
          .format(square_matrix_multiply_dc(A, B)))
    print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<|endoftext|>