<commit_before>import asyncio
from abc import ABC, abstractmethod
from datetime import datetime, timedelta
from enum import Enum, IntEnum, unique
from types import TracebackType
from typing import (
    Any, AsyncContextManager, Awaitable, Callable, Dict, FrozenSet, Generator,
    Iterable, MutableMapping, NamedTuple, Optional, Set, Tuple, Type, TypeVar,
    Union,
)

import aiormq
from aiormq.abc import ExceptionType
from pamqp.common import Arguments
from yarl import URL

from .pool import PoolInstance
from .tools import CallbackCollection, CallbackSetType, CallbackType

TimeoutType = Optional[Union[int, float]]

NoneType = type(None)
DateType = Union[int, datetime, float, timedelta, None]
ExchangeParamType = Union["AbstractExchange", str]
ConsumerTag = str

MILLISECONDS = 1000
ZERO_TIME = datetime.utcfromtimestamp(0)

@unique
class ExchangeType(str, Enum):
    FANOUT = "fanout"
    DIRECT = "direct"
    TOPIC = "topic"
    HEADERS = "headers"
    X_DELAYED_MESSAGE = "x-delayed-message"
    X_CONSISTENT_HASH = "x-consistent-hash"
    X_MODULUS_HASH = "x-modulus-hash"

@unique
class DeliveryMode(IntEnum):
    NOT_PERSISTENT = 1
    PERSISTENT = 2

@unique
class TransactionState(str, Enum):
    CREATED = "created"
    COMMITED = "commited"
    ROLLED_BACK = "rolled back"
    STARTED = "started"

class DeclarationResult(NamedTuple):
    message_count: int
    consumer_count: int

class AbstractTransaction:
    state: TransactionState

    @property
    @abstractmethod
    def channel(self) -> "AbstractChannel":
        raise NotImplementedError

    @abstractmethod
    async def select(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Tx.SelectOk:
        raise NotImplementedError

    @abstractmethod
    async def rollback(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Tx.RollbackOk:
        raise NotImplementedError

    async def commit(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Tx.CommitOk:
        raise NotImplementedError

    async def __aenter__(self) -> "AbstractTransaction":
        raise NotImplementedError

    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

HeadersValue = Union[aiormq.abc.FieldValue, bytes]
HeadersPythonValues = Union[
    HeadersValue,
    Set[HeadersValue],
    Tuple[HeadersValue, ...],
    FrozenSet[HeadersValue],
]
HeadersType = MutableMapping[str, HeadersPythonValues]

class AbstractMessage(ABC):
    body: bytes
    body_size: int
    headers_raw: aiormq.abc.FieldTable
    content_type: Optional[str]
    content_encoding: Optional[str]
    delivery_mode: DeliveryMode
    priority: Optional[int]
    correlation_id: Optional[str]
    reply_to: Optional[str]
    expiration: Optional[DateType]
    message_id: Optional[str]
    timestamp: Optional[datetime]
    type: Optional[str]
    user_id: Optional[str]
    app_id: Optional[str]

    @property
    def headers(self) -> HeadersType:
        raise NotImplementedError

    @headers.setter
    def headers(self, value: HeadersType) -> None:
        raise NotImplementedError

    @abstractmethod
    def info(self) -> Dict[str, HeadersValue]:
        raise NotImplementedError

    @property
    @abstractmethod
    def locked(self) -> bool:
        raise NotImplementedError

    @property
    @abstractmethod
    def properties(self) -> aiormq.spec.Basic.Properties:
        raise NotImplementedError

    @abstractmethod
    def __iter__(self) -> Iterable[bytes]:
        raise NotImplementedError

    @abstractmethod
    def lock(self) -> None:
        raise NotImplementedError

    def __copy__(self) -> "AbstractMessage":
        raise NotImplementedError

class AbstractIncomingMessage(AbstractMessage, ABC):
    cluster_id: Optional[str]
    consumer_tag: Optional["ConsumerTag"]
    delivery_tag: Optional[int]
    redelivered: bool
    message_count: Optional[int]
    routing_key: Optional[str]
    exchange: str

    @property
    @abstractmethod
    def channel(self) -> aiormq.abc.AbstractChannel:
        raise NotImplementedError

    @abstractmethod
    def process(
        self,
        requeue: bool = False,
        reject_on_redelivered: bool = False,
        ignore_processed: bool = False,
    ) -> "AbstractProcessContext":
        raise NotImplementedError

    @abstractmethod
    async def ack(self, multiple: bool = False) -> None:
        raise NotImplementedError

    @abstractmethod
    async def reject(self, requeue: bool = False) -> None:
        raise NotImplementedError

    @abstractmethod
    async def nack(self, multiple: bool = False, requeue: bool = True) -> None:
        raise NotImplementedError

    def info(self) -> Dict[str, Any]:
        raise NotImplementedError

    @property
    @abstractmethod
    def processed(self) -> bool:
        raise NotImplementedError

class AbstractProcessContext(AsyncContextManager):
    @abstractmethod
    async def __aenter__(self) -> AbstractIncomingMessage:
        raise NotImplementedError

    @abstractmethod
    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

class AbstractQueue:
    channel: "AbstractChannel"
    connection: "AbstractConnection"
    name: str
    durable: bool
    exclusive: bool
    auto_delete: bool
    arguments: Arguments
    passive: bool
    declaration_result: aiormq.spec.Queue.DeclareOk

    @abstractmethod
    async def declare(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Queue.DeclareOk:
        raise NotImplementedError

    @abstractmethod
    async def bind(
        self,
        exchange: ExchangeParamType,
        routing_key: str = None,
        *,
        arguments: Arguments = None,
        timeout: TimeoutType = None
    ) -> aiormq.spec.Queue.BindOk:
        raise NotImplementedError

    @abstractmethod
    async def unbind(
        self,
        exchange: ExchangeParamType,
        routing_key: Optional[str] = None,
        arguments: Arguments = None,
        timeout: TimeoutType = None,
    ) -> aiormq.spec.Queue.UnbindOk:
        raise NotImplementedError

    @abstractmethod
    async def consume(
        self,
        callback: Callable[[AbstractIncomingMessage], Any],
        no_ack: bool = False,
        exclusive: bool = False,
        arguments: Arguments = None,
        consumer_tag: ConsumerTag = None,
        timeout: TimeoutType = None,
    ) -> ConsumerTag:
        raise NotImplementedError

    @abstractmethod
    async def cancel(
        self, consumer_tag: ConsumerTag,
        timeout: TimeoutType = None,
        nowait: bool = False,
    ) -> aiormq.spec.Basic.CancelOk:
        raise NotImplementedError

    @abstractmethod
    async def get(
        self, *, no_ack: bool = False,
        fail: bool = True, timeout: TimeoutType = 5
    ) -> Optional[AbstractIncomingMessage]:
        raise NotImplementedError

    @abstractmethod
    async def purge(
        self, no_wait: bool = False, timeout: TimeoutType = None,
    ) -> aiormq.spec.Queue.PurgeOk:
        raise NotImplementedError

    @abstractmethod
    async def delete(
        self, *, if_unused: bool = True, if_empty: bool = True,
        timeout: TimeoutType = None
    ) -> aiormq.spec.Queue.DeleteOk:
        raise NotImplementedError

    @abstractmethod
    def iterator(self, **kwargs: Any) -> "AbstractQueueIterator":
        raise NotImplementedError

class AbstractQueueIterator:
    _amqp_queue: AbstractQueue
    _queue: asyncio.Queue
    _consumer_tag: ConsumerTag
    _consume_kwargs: Dict[str, Any]

    @abstractmethod
    def close(self, *_: Any) -> Awaitable[Any]:
        raise NotImplementedError

    @abstractmethod
    async def on_message(self, message: AbstractIncomingMessage) -> None:
        raise NotImplementedError

    @abstractmethod
    async def consume(self) -> None:
        raise NotImplementedError

    @abstractmethod
    def __aiter__(self) -> "AbstractQueueIterator":
        raise NotImplementedError

    @abstractmethod
    def __aenter__(self) -> Awaitable["AbstractQueueIterator"]:
        raise NotImplementedError

    @abstractmethod
    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

    @abstractmethod
    async def __anext__(self) -> AbstractIncomingMessage:
        raise NotImplementedError

class AbstractExchange(ABC):
    @property
    @abstractmethod
    def channel(self) -> "AbstractChannel":
        raise NotImplementedError

    @abstractmethod
    async def declare(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Exchange.DeclareOk:
        raise NotImplementedError

    @abstractmethod
    async def bind(
        self,
        exchange: ExchangeParamType,
        routing_key: str = "",
        *,
        arguments: Arguments = None,
        timeout: TimeoutType = None
    ) -> aiormq.spec.Exchange.BindOk:
        raise NotImplementedError

    @abstractmethod
    async def unbind(
        self,
        exchange: ExchangeParamType,
        routing_key: str = "",
        arguments: Arguments = None,
        timeout: TimeoutType = None,
    ) -> aiormq.spec.Exchange.UnbindOk:
        raise NotImplementedError

    @abstractmethod
    async def publish(
        self,
        message: "AbstractMessage",
        routing_key: str,
        *,
        mandatory: bool = True,
        immediate: bool = False,
        timeout: TimeoutType = None
    ) -> Optional[aiormq.abc.ConfirmationFrameType]:
        raise NotImplementedError

    @abstractmethod
    async def delete(
        self, if_unused: bool = False, timeout: TimeoutType = None,
    ) -> aiormq.spec.Exchange.DeleteOk:
        raise NotImplementedError

class AbstractChannel(PoolInstance, ABC):
    QUEUE_CLASS: Type[AbstractQueue]
    EXCHANGE_CLASS: Type[AbstractExchange]

    close_callbacks: CallbackCollection
    return_callbacks: CallbackCollection
    connection: "AbstractConnection"
    loop: asyncio.AbstractEventLoop
    default_exchange: AbstractExchange

    @property
    @abstractmethod
    def done_callbacks(self) -> CallbackCollection:
        raise NotImplementedError

    @property
    @abstractmethod
    def is_initialized(self) -> bool:
        return hasattr(self, "_channel")

    @property
    @abstractmethod
    def is_closed(self) -> bool:
        raise NotImplementedError

    @abstractmethod
    def close(self, exc: ExceptionType = None) -> Awaitable[None]:
        raise NotImplementedError

    @property
    @abstractmethod
    def channel(self) -> aiormq.abc.AbstractChannel:
        raise NotImplementedError

    @property
    @abstractmethod
    def number(self) -> Optional[int]:
        raise NotImplementedError

    @abstractmethod
    def __await__(self) -> Generator[Any, Any, "AbstractChannel"]:
        raise NotImplementedError

    @abstractmethod
    async def __aenter__(self) -> "AbstractChannel":
        raise NotImplementedError

    @abstractmethod
    def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> Awaitable[None]:
        raise NotImplementedError

    @abstractmethod
    async def initialize(self, timeout: TimeoutType = None) -> None:
        raise NotImplementedError

    @abstractmethod
    def reopen(self) -> Awaitable[None]:
        raise NotImplementedError

    @abstractmethod
    async def declare_exchange(
        self,
        name: str,
        type: Union[ExchangeType, str] = ExchangeType.DIRECT,
        *,
        durable: bool = False,
        auto_delete: bool = False,
        internal: bool = False,
        passive: bool = False,
        arguments: Arguments = None,
        timeout: TimeoutType = None,
    ) -> AbstractExchange:
        raise NotImplementedError

    @abstractmethod
    async def get_exchange(
        self, name: str, *, ensure: bool = True
    ) -> AbstractExchange:
        raise NotImplementedError

    @abstractmethod
    async def declare_queue(
        self,
        name: str = None,
        *,
        durable: bool = False,
        exclusive: bool = False,
        passive: bool = False,
        auto_delete: bool = False,
        arguments: Arguments = None,
        timeout: TimeoutType = None
    ) -> AbstractQueue:
        raise NotImplementedError

    @abstractmethod
    async def get_queue(
        self, name: str, *, ensure: bool = True
    ) -> AbstractQueue:
        raise NotImplementedError

    @abstractmethod
    async def set_qos(
        self,
        prefetch_count: int = 0,
        prefetch_size: int = 0,
        global_: bool = False,
        timeout: TimeoutType = None,
        all_channels: bool = None,
    ) -> aiormq.spec.Basic.QosOk:
        raise NotImplementedError

    @abstractmethod
    async def queue_delete(
        self,
        queue_name: str,
        timeout: TimeoutType = None,
        if_unused: bool = False,
        if_empty: bool = False,
        nowait: bool = False,
    ) -> aiormq.spec.Queue.DeleteOk:
        raise NotImplementedError

    @abstractmethod
    async def exchange_delete(
        self,
        exchange_name: str,
        timeout: TimeoutType = None,
        if_unused: bool = False,
        nowait: bool = False,
    ) -> aiormq.spec.Exchange.DeleteOk:
        raise NotImplementedError

    @abstractmethod
    def transaction(self) -> AbstractTransaction:
        raise NotImplementedError

    @abstractmethod
    async def flow(self, active: bool = True) -> aiormq.spec.Channel.FlowOk:
        raise NotImplementedError

class AbstractConnection(PoolInstance, ABC):
    loop: asyncio.AbstractEventLoop
    close_callbacks: CallbackCollection
    connected: asyncio.Event
    connection: aiormq.abc.AbstractConnection

    @abstractmethod
    def __init__(
        self, url: URL, loop: Optional[asyncio.AbstractEventLoop] = None,
        **kwargs: Any
    ):
        pass

    @property
    @abstractmethod
    def is_closed(self) -> bool:
        raise NotImplementedError

    @abstractmethod
    async def close(self, exc: ExceptionType = asyncio.CancelledError) -> None:
        raise NotImplementedError

    @abstractmethod
    async def connect(
        self, timeout: TimeoutType = None, **kwargs: Any
    ) -> None:
        raise NotImplementedError

    @abstractmethod
    def channel(
        self,
        channel_number: int = None,
        publisher_confirms: bool = True,
        on_return_raises: bool = False,
    ) -> AbstractChannel:
        raise NotImplementedError

    @abstractmethod
    async def ready(self) -> None:
        raise NotImplementedError

    @abstractmethod
    async def __aenter__(self) -> "AbstractConnection":
        raise NotImplementedError

    @abstractmethod
    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

class AbstractRobustQueue(AbstractQueue):
    @abstractmethod
    def reopen(self) -> Awaitable[None]:
        raise NotImplementedError

class AbstractRobustExchange(AbstractExchange):
    @abstractmethod
    def restore(self, channel: "AbstractRobustChannel") -> Awaitable[None]:
        raise NotImplementedError

class AbstractRobustChannel(AbstractChannel):
    reopen_callbacks: CallbackCollection

    @abstractmethod
    def reopen(self) -> Awaitable[None]:
        raise NotImplementedError

    @abstractmethod
    def restore(self) -> Awaitable[None]:
        raise NotImplementedError

class AbstractRobustConnection(AbstractConnection):
    @property
    @abstractmethod
    def reconnecting(self) -> bool:
        raise NotImplementedError

    @property
    @abstractmethod
    def reconnect_callbacks(self) -> CallbackCollection:
        raise NotImplementedError

    @abstractmethod
    def reconnect(self) -> Awaitable[None]:
        raise NotImplementedError

ChannelCloseCallback = Callable[[AbstractChannel, Any], Any]
ConnectionCloseCallback = Callable[[AbstractConnection], Any]
ConnectionType = TypeVar("ConnectionType", bound=AbstractConnection)

__all__ = (
    "AbstractChannel",
    "AbstractConnection",
    "AbstractExchange",
    "AbstractIncomingMessage",
    "AbstractMessage",
    "AbstractProcessContext",
    "AbstractQueue",
    "AbstractQueueIterator",
    "AbstractRobustChannel",
    "AbstractRobustConnection",
    "AbstractRobustExchange",
    "AbstractRobustQueue",
    "AbstractTransaction",
    "CallbackSetType",
    "CallbackType",
    "ChannelCloseCallback",
    "ConnectionCloseCallback",
    "ConsumerTag",
    "DateType",
    "DeclarationResult",
    "DeliveryMode",
    "ExchangeParamType",
    "ExchangeType",
    "HeadersPythonValues",
    "HeadersType",
    "HeadersValue",
    "MILLISECONDS",
    "TimeoutType",
    "TransactionState",
    "ZERO_TIME",
)
<commit_msg>missed abstractmethod<commit_after>import asyncio
from abc import ABC, abstractmethod
from datetime import datetime, timedelta
from enum import Enum, IntEnum, unique
from types import TracebackType
from typing import (
    Any, AsyncContextManager, Awaitable, Callable, Dict, FrozenSet, Generator,
    Iterable, MutableMapping, NamedTuple, Optional, Set, Tuple, Type, TypeVar,
    Union,
)

import aiormq
from aiormq.abc import ExceptionType
from pamqp.common import Arguments
from yarl import URL

from .pool import PoolInstance
from .tools import CallbackCollection, CallbackSetType, CallbackType

TimeoutType = Optional[Union[int, float]]

NoneType = type(None)
DateType = Union[int, datetime, float, timedelta, None]
ExchangeParamType = Union["AbstractExchange", str]
ConsumerTag = str

MILLISECONDS = 1000
ZERO_TIME = datetime.utcfromtimestamp(0)

@unique
class ExchangeType(str, Enum):
    FANOUT = "fanout"
    DIRECT = "direct"
    TOPIC = "topic"
    HEADERS = "headers"
    X_DELAYED_MESSAGE = "x-delayed-message"
    X_CONSISTENT_HASH = "x-consistent-hash"
    X_MODULUS_HASH = "x-modulus-hash"

@unique
class DeliveryMode(IntEnum):
    NOT_PERSISTENT = 1
    PERSISTENT = 2

@unique
class TransactionState(str, Enum):
    CREATED = "created"
    COMMITED = "commited"
    ROLLED_BACK = "rolled back"
    STARTED = "started"

class DeclarationResult(NamedTuple):
    message_count: int
    consumer_count: int

class AbstractTransaction:
    state: TransactionState

    @property
    @abstractmethod
    def channel(self) -> "AbstractChannel":
        raise NotImplementedError

    @abstractmethod
    async def select(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Tx.SelectOk:
        raise NotImplementedError

    @abstractmethod
    async def rollback(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Tx.RollbackOk:
        raise NotImplementedError

    async def commit(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Tx.CommitOk:
        raise NotImplementedError

    async def __aenter__(self) -> "AbstractTransaction":
        raise NotImplementedError

    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

HeadersValue = Union[aiormq.abc.FieldValue, bytes]
HeadersPythonValues = Union[
    HeadersValue,
    Set[HeadersValue],
    Tuple[HeadersValue, ...],
    FrozenSet[HeadersValue],
]
HeadersType = MutableMapping[str, HeadersPythonValues]

class AbstractMessage(ABC):
    body: bytes
    body_size: int
    headers_raw: aiormq.abc.FieldTable
    content_type: Optional[str]
    content_encoding: Optional[str]
    delivery_mode: DeliveryMode
    priority: Optional[int]
    correlation_id: Optional[str]
    reply_to: Optional[str]
    expiration: Optional[DateType]
    message_id: Optional[str]
    timestamp: Optional[datetime]
    type: Optional[str]
    user_id: Optional[str]
    app_id: Optional[str]

    @property
    @abstractmethod
    def headers(self) -> HeadersType:
        raise NotImplementedError

    @headers.setter
    def headers(self, value: HeadersType) -> None:
        raise NotImplementedError

    @abstractmethod
    def info(self) -> Dict[str, HeadersValue]:
        raise NotImplementedError

    @property
    @abstractmethod
    def locked(self) -> bool:
        raise NotImplementedError

    @property
    @abstractmethod
    def properties(self) -> aiormq.spec.Basic.Properties:
        raise NotImplementedError

    @abstractmethod
    def __iter__(self) -> Iterable[bytes]:
        raise NotImplementedError

    @abstractmethod
    def lock(self) -> None:
        raise NotImplementedError

    def __copy__(self) -> "AbstractMessage":
        raise NotImplementedError

class AbstractIncomingMessage(AbstractMessage, ABC):
    cluster_id: Optional[str]
    consumer_tag: Optional["ConsumerTag"]
    delivery_tag: Optional[int]
    redelivered: bool
    message_count: Optional[int]
    routing_key: Optional[str]
    exchange: str

    @property
    @abstractmethod
    def channel(self) -> aiormq.abc.AbstractChannel:
        raise NotImplementedError

    @abstractmethod
    def process(
        self,
        requeue: bool = False,
        reject_on_redelivered: bool = False,
        ignore_processed: bool = False,
    ) -> "AbstractProcessContext":
        raise NotImplementedError

    @abstractmethod
    async def ack(self, multiple: bool = False) -> None:
        raise NotImplementedError

    @abstractmethod
    async def reject(self, requeue: bool = False) -> None:
        raise NotImplementedError

    @abstractmethod
    async def nack(self, multiple: bool = False, requeue: bool = True) -> None:
        raise NotImplementedError

    def info(self) -> Dict[str, Any]:
        raise NotImplementedError

    @property
    @abstractmethod
    def processed(self) -> bool:
        raise NotImplementedError

class AbstractProcessContext(AsyncContextManager):
    @abstractmethod
    async def __aenter__(self) -> AbstractIncomingMessage:
        raise NotImplementedError

    @abstractmethod
    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

class AbstractQueue:
    channel: "AbstractChannel"
    connection: "AbstractConnection"
    name: str
    durable: bool
    exclusive: bool
    auto_delete: bool
    arguments: Arguments
    passive: bool
    declaration_result: aiormq.spec.Queue.DeclareOk

    @abstractmethod
    async def declare(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Queue.DeclareOk:
        raise NotImplementedError

    @abstractmethod
    async def bind(
        self,
        exchange: ExchangeParamType,
        routing_key: str = None,
        *,
        arguments: Arguments = None,
        timeout: TimeoutType = None
    ) -> aiormq.spec.Queue.BindOk:
        raise NotImplementedError

    @abstractmethod
    async def unbind(
        self,
        exchange: ExchangeParamType,
        routing_key: Optional[str] = None,
        arguments: Arguments = None,
        timeout: TimeoutType = None,
    ) -> aiormq.spec.Queue.UnbindOk:
        raise NotImplementedError

    @abstractmethod
    async def consume(
        self,
        callback: Callable[[AbstractIncomingMessage], Any],
        no_ack: bool = False,
        exclusive: bool = False,
        arguments: Arguments = None,
        consumer_tag: ConsumerTag = None,
        timeout: TimeoutType = None,
    ) -> ConsumerTag:
        raise NotImplementedError

    @abstractmethod
    async def cancel(
        self, consumer_tag: ConsumerTag,
        timeout: TimeoutType = None,
        nowait: bool = False,
    ) -> aiormq.spec.Basic.CancelOk:
        raise NotImplementedError

    @abstractmethod
    async def get(
        self, *, no_ack: bool = False,
        fail: bool = True, timeout: TimeoutType = 5
    ) -> Optional[AbstractIncomingMessage]:
        raise NotImplementedError

    @abstractmethod
    async def purge(
        self, no_wait: bool = False, timeout: TimeoutType = None,
    ) -> aiormq.spec.Queue.PurgeOk:
        raise NotImplementedError

    @abstractmethod
    async def delete(
        self, *, if_unused: bool = True, if_empty: bool = True,
        timeout: TimeoutType = None
    ) -> aiormq.spec.Queue.DeleteOk:
        raise NotImplementedError

    @abstractmethod
    def iterator(self, **kwargs: Any) -> "AbstractQueueIterator":
        raise NotImplementedError

class AbstractQueueIterator:
    _amqp_queue: AbstractQueue
    _queue: asyncio.Queue
    _consumer_tag: ConsumerTag
    _consume_kwargs: Dict[str, Any]

    @abstractmethod
    def close(self, *_: Any) -> Awaitable[Any]:
        raise NotImplementedError

    @abstractmethod
    async def on_message(self, message: AbstractIncomingMessage) -> None:
        raise NotImplementedError

    @abstractmethod
    async def consume(self) -> None:
        raise NotImplementedError

    @abstractmethod
    def __aiter__(self) -> "AbstractQueueIterator":
        raise NotImplementedError

    @abstractmethod
    def __aenter__(self) -> Awaitable["AbstractQueueIterator"]:
        raise NotImplementedError

    @abstractmethod
    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

    @abstractmethod
    async def __anext__(self) -> AbstractIncomingMessage:
        raise NotImplementedError

class AbstractExchange(ABC):
    @property
    @abstractmethod
    def channel(self) -> "AbstractChannel":
        raise NotImplementedError

    @abstractmethod
    async def declare(
        self, timeout: TimeoutType = None,
    ) -> aiormq.spec.Exchange.DeclareOk:
        raise NotImplementedError

    @abstractmethod
    async def bind(
        self,
        exchange: ExchangeParamType,
        routing_key: str = "",
        *,
        arguments: Arguments = None,
        timeout: TimeoutType = None
    ) -> aiormq.spec.Exchange.BindOk:
        raise NotImplementedError

    @abstractmethod
    async def unbind(
        self,
        exchange: ExchangeParamType,
        routing_key: str = "",
        arguments: Arguments = None,
        timeout: TimeoutType = None,
    ) -> aiormq.spec.Exchange.UnbindOk:
        raise NotImplementedError

    @abstractmethod
    async def publish(
        self,
        message: "AbstractMessage",
        routing_key: str,
        *,
        mandatory: bool = True,
        immediate: bool = False,
        timeout: TimeoutType = None
    ) -> Optional[aiormq.abc.ConfirmationFrameType]:
        raise NotImplementedError

    @abstractmethod
    async def delete(
        self, if_unused: bool = False, timeout: TimeoutType = None,
    ) -> aiormq.spec.Exchange.DeleteOk:
        raise NotImplementedError

class AbstractChannel(PoolInstance, ABC):
    QUEUE_CLASS: Type[AbstractQueue]
    EXCHANGE_CLASS: Type[AbstractExchange]

    close_callbacks: CallbackCollection
    return_callbacks: CallbackCollection
    connection: "AbstractConnection"
    loop: asyncio.AbstractEventLoop
    default_exchange: AbstractExchange

    @property
    @abstractmethod
    def done_callbacks(self) -> CallbackCollection:
        raise NotImplementedError

    @property
    @abstractmethod
    def is_initialized(self) -> bool:
        return hasattr(self, "_channel")

    @property
    @abstractmethod
    def is_closed(self) -> bool:
        raise NotImplementedError

    @abstractmethod
    def close(self, exc: ExceptionType = None) -> Awaitable[None]:
        raise NotImplementedError

    @property
    @abstractmethod
    def channel(self) -> aiormq.abc.AbstractChannel:
        raise NotImplementedError

    @property
    @abstractmethod
    def number(self) -> Optional[int]:
        raise NotImplementedError

    @abstractmethod
    def __await__(self) -> Generator[Any, Any, "AbstractChannel"]:
        raise NotImplementedError

    @abstractmethod
    async def __aenter__(self) -> "AbstractChannel":
        raise NotImplementedError

    @abstractmethod
    def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> Awaitable[None]:
        raise NotImplementedError

    @abstractmethod
    async def initialize(self, timeout: TimeoutType = None) -> None:
        raise NotImplementedError

    @abstractmethod
    def reopen(self) -> Awaitable[None]:
        raise NotImplementedError

    @abstractmethod
    async def declare_exchange(
        self,
        name: str,
        type: Union[ExchangeType, str] = ExchangeType.DIRECT,
        *,
        durable: bool = False,
        auto_delete: bool = False,
        internal: bool = False,
        passive: bool = False,
        arguments: Arguments = None,
        timeout: TimeoutType = None,
    ) -> AbstractExchange:
        raise NotImplementedError

    @abstractmethod
    async def get_exchange(
        self, name: str, *, ensure: bool = True
    ) -> AbstractExchange:
        raise NotImplementedError

    @abstractmethod
    async def declare_queue(
        self,
        name: str = None,
        *,
        durable: bool = False,
        exclusive: bool = False,
        passive: bool = False,
        auto_delete: bool = False,
        arguments: Arguments = None,
        timeout: TimeoutType = None
    ) -> AbstractQueue:
        raise NotImplementedError

    @abstractmethod
    async def get_queue(
        self, name: str, *, ensure: bool = True
    ) -> AbstractQueue:
        raise NotImplementedError

    @abstractmethod
    async def set_qos(
        self,
        prefetch_count: int = 0,
        prefetch_size: int = 0,
        global_: bool = False,
        timeout: TimeoutType = None,
        all_channels: bool = None,
    ) -> aiormq.spec.Basic.QosOk:
        raise NotImplementedError

    @abstractmethod
    async def queue_delete(
        self,
        queue_name: str,
        timeout: TimeoutType = None,
        if_unused: bool = False,
        if_empty: bool = False,
        nowait: bool = False,
    ) -> aiormq.spec.Queue.DeleteOk:
        raise NotImplementedError

    @abstractmethod
    async def exchange_delete(
        self,
        exchange_name: str,
        timeout: TimeoutType = None,
        if_unused: bool = False,
        nowait: bool = False,
    ) -> aiormq.spec.Exchange.DeleteOk:
        raise NotImplementedError

    @abstractmethod
    def transaction(self) -> AbstractTransaction:
        raise NotImplementedError

    @abstractmethod
    async def flow(self, active: bool = True) -> aiormq.spec.Channel.FlowOk:
        raise NotImplementedError

class AbstractConnection(PoolInstance, ABC):
    loop: asyncio.AbstractEventLoop
    close_callbacks: CallbackCollection
    connected: asyncio.Event
    connection: aiormq.abc.AbstractConnection

    @abstractmethod
    def __init__(
        self, url: URL, loop: Optional[asyncio.AbstractEventLoop] = None,
        **kwargs: Any
    ):
        pass

    @property
    @abstractmethod
    def is_closed(self) -> bool:
        raise NotImplementedError

    @abstractmethod
    async def close(self, exc: ExceptionType = asyncio.CancelledError) -> None:
        raise NotImplementedError

    @abstractmethod
    async def connect(
        self, timeout: TimeoutType = None, **kwargs: Any
    ) -> None:
        raise NotImplementedError

    @abstractmethod
    def channel(
        self,
        channel_number: int = None,
        publisher_confirms: bool = True,
        on_return_raises: bool = False,
    ) -> AbstractChannel:
        raise NotImplementedError

    @abstractmethod
    async def ready(self) -> None:
        raise NotImplementedError

    @abstractmethod
    async def __aenter__(self) -> "AbstractConnection":
        raise NotImplementedError

    @abstractmethod
    async def __aexit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        raise NotImplementedError

class AbstractRobustQueue(AbstractQueue):
    @abstractmethod
    def reopen(self) -> Awaitable[None]:
        raise NotImplementedError

class AbstractRobustExchange(AbstractExchange):
    @abstractmethod
    def restore(self, channel: "AbstractRobustChannel") -> Awaitable[None]:
        raise NotImplementedError

class AbstractRobustChannel(AbstractChannel):
    reopen_callbacks: CallbackCollection

    @abstractmethod
    def reopen(self) -> Awaitable[None]:
        raise NotImplementedError

    @abstractmethod
    def restore(self) -> Awaitable[None]:
        raise NotImplementedError

class AbstractRobustConnection(AbstractConnection):
    @property
    @abstractmethod
    def reconnecting(self) -> bool:
        raise NotImplementedError

    @property
    @abstractmethod
    def reconnect_callbacks(self) -> CallbackCollection:
        raise NotImplementedError

    @abstractmethod
    def reconnect(self) -> Awaitable[None]:
        raise NotImplementedError

ChannelCloseCallback = Callable[[AbstractChannel, Any], Any]
ConnectionCloseCallback = Callable[[AbstractConnection], Any]
ConnectionType = TypeVar("ConnectionType", bound=AbstractConnection)

__all__ = (
    "AbstractChannel",
    "AbstractConnection",
    "AbstractExchange",
    "AbstractIncomingMessage",
    "AbstractMessage",
    "AbstractProcessContext",
    "AbstractQueue",
    "AbstractQueueIterator",
    "AbstractRobustChannel",
    "AbstractRobustConnection",
    "AbstractRobustExchange",
    "AbstractRobustQueue",
    "AbstractTransaction",
    "CallbackSetType",
    "CallbackType",
    "ChannelCloseCallback",
    "ConnectionCloseCallback",
    "ConsumerTag",
    "DateType",
    "DeclarationResult",
    "DeliveryMode",
    "ExchangeParamType",
    "ExchangeType",
    "HeadersPythonValues",
    "HeadersType",
    "HeadersValue",
    "MILLISECONDS",
    "TimeoutType",
    "TransactionState",
    "ZERO_TIME",
)
<|endoftext|>


<commit_before>import sublime, sublime_plugin
import os

# todo:
# * fix lag
# * glob modules subfolder for languages and dynamically load - remove the current ugly hardcodedness

import sublimelint.modules.python as python

drawType = 4 | 32

languages = [python]

global lineMessages
lineMessages = {}

def run(module, view):
	global lineMessages
	vid = view.id()

	text = view.substr(sublime.Region(0, view.size()))

	if view.file_name():
		filename = os.path.split(view.file_name())[-1]
	else:
		filename = 'untitled'

	underline, lines, errorMessages, clearOutlines = module.run(text, view, filename)
	lineMessages[vid] = errorMessages

	view.erase_regions('lint-syntax')
	view.erase_regions('lint-syntax-underline')
	view.erase_regions('lint-underline')

	if clearOutlines:
		view.erase_regions('lint-outlines')

	if underline:
		view.add_regions('lint-underline', underline, 'keyword', drawType)#sublime.DRAW_EMPTY_AS_OVERWRITE | sublime.DRAW_OUTLINED)

	if lines:
		outlines = [view.full_line(view.text_point(lineno, 0)) for lineno in lines]
		view.add_regions('lint-outlines', outlines, 'keyword', drawType)#sublime.DRAW_EMPTY_AS_OVERWRITE | sublime.DRAW_OUTLINED)

def validate(view):
	for module in languages:
		if module.language in view.settings().get("syntax"):
			run(module, view)
			break

import time, thread
global queue, lookup
queue = {}
lookup = {}

def validate_runner(): # this threaded runner keeps it from slowing down UI while you type
	global queue, lookup
	while True:
		time.sleep(0.5)
		for vid in dict(queue):
			if queue[vid] == 0:
				v = lookup[vid]
				def _view():
					try:
						validate(v)
					except RuntimeError, excp:
						print excp
				sublime.set_timeout(_view, 100)
				try: del queue[vid]
				except: pass
				try: del lookup[vid]
				except: pass
			else:
				queue[vid] = 0

def validate_hit(view):
	global lookup
	global queue

	for module in languages:
		if module.language in view.settings().get("syntax"):
			break
	else:
		view.erase_regions('lint-syntax')
		view.erase_regions('lint-syntax-underline')
		view.erase_regions('lint-underline')
		view.erase_regions('lint-outlines')
		return

	vid = view.id()
	lookup[vid] = view
	queue[vid] = 1

thread.start_new_thread(validate_runner, ())

class pyflakes(sublime_plugin.EventListener):
	def __init__(self, *args, **kwargs):
		sublime_plugin.EventListener.__init__(self, *args, **kwargs)
		self.lastCount = {}

	def on_modified(self, view):
		validate_hit(view)
		return

		# alternate method which works alright when we don't have threads/set_timeout
		# from when I ported to early X beta :P
		text = view.substr(sublime.Region(0, view.size()))
		count = text.count('\n')
		if count > 500: return
		bid = view.buffer_id()

		if bid in self.lastCount:
			if self.lastCount[bid] != count:
				validate(view)

		self.lastCount[bid] = count

	def on_load(self, view):
		validate_hit(view)

	def on_post_save(self, view):
		validate_hit(view)

	def on_selection_modified(self, view):
		vid = view.id()
		lineno = view.rowcol(view.sel()[0].end())[0]
		if vid in lineMessages and lineno in lineMessages[vid]:
			view.set_status('pyflakes', '; '.join(lineMessages[vid][lineno]))
		else:
			view.erase_status('pyflakes')
<commit_msg>force syntax checks to run on initial load<commit_after>import sublime, sublime_plugin
import os

# todo:
# * fix lag
# * glob modules subfolder for languages and dynamically load - remove the current ugly hardcodedness

import sublimelint.modules.python as python

drawType = 4 | 32

languages = [python]

global lineMessages
lineMessages = {}

def run(module, view):
	global lineMessages
	vid = view.id()

	text = view.substr(sublime.Region(0, view.size()))

	if view.file_name():
		filename = os.path.split(view.file_name())[-1]
	else:
		filename = 'untitled'

	underline, lines, errorMessages, clearOutlines = module.run(text, view, filename)
	lineMessages[vid] = errorMessages

	view.erase_regions('lint-syntax')
	view.erase_regions('lint-syntax-underline')
	view.erase_regions('lint-underline')

	if clearOutlines:
		view.erase_regions('lint-outlines')

	if underline:
		view.add_regions('lint-underline', underline, 'keyword', drawType)#sublime.DRAW_EMPTY_AS_OVERWRITE | sublime.DRAW_OUTLINED)

	if lines:
		outlines = [view.full_line(view.text_point(lineno, 0)) for lineno in lines]
		view.add_regions('lint-outlines', outlines, 'keyword', drawType)#sublime.DRAW_EMPTY_AS_OVERWRITE | sublime.DRAW_OUTLINED)

def validate(view):
	for module in languages:
		if module.language in view.settings().get("syntax"):
			run(module, view)
			break

import time, thread
global queue, lookup
queue = {}
lookup = {}

def validate_runner(): # this threaded runner keeps it from slowing down UI while you type
	global queue, lookup
	while True:
		time.sleep(0.5)
		for vid in dict(queue):
			if queue[vid] == 0:
				v = lookup[vid]
				def _view():
					try:
						validate(v)
					except RuntimeError, excp:
						print excp
				sublime.set_timeout(_view, 100)
				try: del queue[vid]
				except: pass
				try: del lookup[vid]
				except: pass
			else:
				queue[vid] = 0

def validate_hit(view):
	global lookup
	global queue

	for module in languages:
		if module.language in view.settings().get("syntax"):
			break
	else:
		view.erase_regions('lint-syntax')
		view.erase_regions('lint-syntax-underline')
		view.erase_regions('lint-underline')
		view.erase_regions('lint-outlines')
		return

	vid = view.id()
	lookup[vid] = view
	queue[vid] = 1

thread.start_new_thread(validate_runner, ())

class pyflakes(sublime_plugin.EventListener):
	def __init__(self, *args, **kwargs):
		sublime_plugin.EventListener.__init__(self, *args, **kwargs)
		self.lastCount = {}

	def on_modified(self, view):
		validate_hit(view)
		return

		# alternate method which works alright when we don't have threads/set_timeout
		# from when I ported to early X beta :P
		text = view.substr(sublime.Region(0, view.size()))
		count = text.count('\n')
		if count > 500: return
		bid = view.buffer_id()

		if bid in self.lastCount:
			if self.lastCount[bid] != count:
				validate(view)

		self.lastCount[bid] = count

	def on_load(self, view):
		validate(view)

	def on_post_save(self, view):
		validate_hit(view)

	def on_selection_modified(self, view):
		vid = view.id()
		lineno = view.rowcol(view.sel()[0].end())[0]
		if vid in lineMessages and lineno in lineMessages[vid]:
			view.set_status('pyflakes', '; '.join(lineMessages[vid][lineno]))
		else:
			view.erase_status('pyflakes')
<|endoftext|>


<commit_before>from django.conf.urls import patterns, url
from alapage.views import HomepageView, PageView

urlpatterns = patterns('',
    url(r'^(?P<url>.*?)$', PageView.as_view(), name="page-view"),
    url(r'^', HomepageView.as_view(), name="home-view"),
)

<commit_msg>New Django 1.9 url format<commit_after>from django.conf.urls import url
from alapage.views import HomepageView, PageView

urlpatterns = [
    url(r'^(?P<url>.*?)$', PageView.as_view(), name="page-view"),
    url(r'^', HomepageView.as_view(), name="home-view"),
    ]

<|endoftext|>


<commit_before># encoding: utf-8
#----------------------------------------------------------------------------
#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
#  Distributed under the terms of the BSD License.  See COPYING.rst.
#----------------------------------------------------------------------------

"""
Plotting functions for distarrays.
"""

from matplotlib import pyplot, colors, cm
from numpy import arange, concatenate, linspace, resize

from distarray.decorators import local

@local
def _get_ranks(arr):
    """
    Given a distarray arr, return a distarray with the same shape, but
    with the elements equal to the rank of the process the element is
    on.
    """
    out = arr.copy()
    out.local_array[:] = arr.comm_rank
    out.local_array = out.local_array.astype(int)
    return out

def cmap_discretize(cmap, N):
    """Create a discrete colormap from the continuous colormap cmap.

    Parameters
    ----------
    cmap : colormap instance, or string
        The continuous colormap, as object or name, to make discrete.
        For example, matplotlib.cm.jet, or 'jet'.
    N : int
        The number of discrete colors desired.

    Returns
    -------
    colormap
        The desired discrete colormap.

    Example usage:
    >>> x = resize(arange(100), (5,100))
    >>> djet = cmap_discretize(cm.jet, 5)
    >>> pyplot.imshow(x, cmap=djet)
    """
    # This is copied from:
    # http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
    if type(cmap) == str:
        cmap = cm.get_cmap(cmap)
    colors_i = concatenate((linspace(0, 1., N), (0., 0., 0., 0.)))
    colors_rgba = cmap(colors_i)
    indices = linspace(0, 1., N + 1)
    cdict = {}
    for ki, key in enumerate(('red', 'green', 'blue')):
        cdict[key] = [
            (indices[i], colors_rgba[i - 1, ki], colors_rgba[i, ki])
            for i in xrange(N + 1)
        ]
    # Return colormap object.
    return colors.LinearSegmentedColormap(cmap.name + "_%d" % N, cdict, 1024)

def plot_array_distribution(darray,
                            title=None,
                            xlabel=None,
                            ylabel=None,
                            yflip=False,
                            legend=False,
                            filename=None,
                            *args, **kwargs):
    """
    Plot a distarray's memory layout. It can be 1D or 2D.
    Elements are colored according to the process they are on.

    Parameters
    ----------
    title : string
        Text label for the plot title, or None.
    xlabel : string
        Text label for the x-axis, or None.
    ylabel : string
        Text label for the y-axis, or None.
    yflip : bool
        If True, then the y-axis increases downwards, to match the layout
        when printing the array itself.
    legend : bool
        If True, then a colorbar legend is drawn to label the colors.
    filename : string
        Output filename for the plot image.

    Returns
    -------
    out
        The process assignment array, as a DistArray.
    """
    # This is based somewhat on:
    #   http://matplotlib.org/examples/api/colorbar_only.html

    # Process per element.
    process_darray = _get_ranks(darray)
    process_array = process_darray.toarray()

    # Values per element.
    values_array = darray.toarray()

    # Coerce to 2D if needed.
    if len(process_array.shape) == 1:
        process_array.shape = (1, process_array.shape[0])
        values_array.shape = process_array.shape

    # Create discrete colormap.
    num_processors = process_array.max() + 1
    cmap = cmap_discretize(cm.jet, num_processors)
    bounds = range(num_processors + 1)
    norm = colors.BoundaryNorm(bounds, cmap.N)

    # Plot the array.
    img = pyplot.matshow(process_array, cmap=cmap, norm=norm, *args, **kwargs)

    # Add title and labels.
    if title is not None:
        pyplot.title(title)
    if xlabel is not None:
        pyplot.xlabel(xlabel)
    if ylabel is not None:
        pyplot.ylabel(ylabel)

    # Either invert y-axis, and put tick labels at bottom,
    # or put the x-axis label at the top.
    axis = pyplot.gca()
    if yflip:
        axis.invert_yaxis()
        for tick in axis.xaxis.iter_ticks():
            tick[0].label1On = True
            tick[0].label2On = False
    else:
        axis.xaxis.set_label_position('top')

    # Label each cell.
    for row in range(values_array.shape[0]):
        for col in range(values_array.shape[1]):
            process = process_array[row, col]
            value = values_array[row, col]
            label = '%d' % (value)
            color = 'white'
            if process == 2:
                # Black on yellow is more readable.
                # Assumes that process=2 is colored yellow!
                color = 'black'
            pyplot.text(
                col, row, label,
                horizontalalignment='center',
                verticalalignment='center',
                color=color)

    # Add colorbar legend.
    if legend:
        cbar = pyplot.colorbar(img)
        cbar_ticks = [0.5 + p for p in range(num_processors)]
        cbar_labels = [str(p) for p in range(num_processors)]
        cbar.set_ticks(cbar_ticks)
        cbar.set_ticklabels(cbar_labels)
        cbar.set_label('Processor')

    # Adjust size.
    figure = pyplot.gcf()
    figure.set_size_inches(10.0, 5.0)

    # Save to output file.
    if filename is not None:
        pyplot.savefig(filename, dpi=100)

    return process_darray
<commit_msg>Smarter text color selection.<commit_after># encoding: utf-8
#----------------------------------------------------------------------------
#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
#  Distributed under the terms of the BSD License.  See COPYING.rst.
#----------------------------------------------------------------------------

"""
Plotting functions for distarrays.
"""

from matplotlib import pyplot, colors, cm
from numpy import arange, concatenate, linspace, resize

from distarray.decorators import local

@local
def _get_ranks(arr):
    """
    Given a distarray arr, return a distarray with the same shape, but
    with the elements equal to the rank of the process the element is
    on.
    """
    out = arr.copy()
    out.local_array[:] = arr.comm_rank
    out.local_array = out.local_array.astype(int)
    return out

def cmap_discretize(cmap, N):
    """Create a discrete colormap from the continuous colormap cmap.

    Parameters
    ----------
    cmap : colormap instance, or string
        The continuous colormap, as object or name, to make discrete.
        For example, matplotlib.cm.jet, or 'jet'.
    N : int
        The number of discrete colors desired.

    Returns
    -------
    colormap
        The desired discrete colormap.

    Example usage:
    >>> x = resize(arange(100), (5,100))
    >>> djet = cmap_discretize(cm.jet, 5)
    >>> pyplot.imshow(x, cmap=djet)
    """
    # This is copied from:
    # http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
    if type(cmap) == str:
        cmap = cm.get_cmap(cmap)
    colors_i = concatenate((linspace(0, 1., N), (0., 0., 0., 0.)))
    colors_rgba = cmap(colors_i)
    indices = linspace(0, 1., N + 1)
    cdict = {}
    for ki, key in enumerate(('red', 'green', 'blue')):
        cdict[key] = [
            (indices[i], colors_rgba[i - 1, ki], colors_rgba[i, ki])
            for i in xrange(N + 1)
        ]
    # Return colormap object.
    return colors.LinearSegmentedColormap(cmap.name + "_%d" % N, cdict, 1024)

def plot_array_distribution(darray,
                            title=None,
                            xlabel=None,
                            ylabel=None,
                            yflip=False,
                            legend=False,
                            filename=None,
                            *args, **kwargs):
    """
    Plot a distarray's memory layout. It can be 1D or 2D.
    Elements are colored according to the process they are on.

    Parameters
    ----------
    title : string
        Text label for the plot title, or None.
    xlabel : string
        Text label for the x-axis, or None.
    ylabel : string
        Text label for the y-axis, or None.
    yflip : bool
        If True, then the y-axis increases downwards, to match the layout
        when printing the array itself.
    legend : bool
        If True, then a colorbar legend is drawn to label the colors.
    filename : string
        Output filename for the plot image.

    Returns
    -------
    out
        The process assignment array, as a DistArray.
    """
    # This is based somewhat on:
    #   http://matplotlib.org/examples/api/colorbar_only.html

    # Process per element.
    process_darray = _get_ranks(darray)
    process_array = process_darray.toarray()

    # Values per element.
    values_array = darray.toarray()

    # Coerce to 2D if needed.
    if len(process_array.shape) == 1:
        process_array.shape = (1, process_array.shape[0])
        values_array.shape = process_array.shape

    # Create discrete colormap.
    num_processors = process_array.max() + 1
    cmap = cmap_discretize(cm.jet, num_processors)
    bounds = range(num_processors + 1)
    norm = colors.BoundaryNorm(bounds, cmap.N)

    # Choose a text color for each processor index.
    # The idea is to pick black for colors near white.
    # This is not sophisticated but ok for this use.
    text_colors = []
    for j in range(num_processors):
        # Get rgb color that matshow() will use.
        jj = float(j + 0.5) / float(num_processors)
        cj = cmap(jj)
        # Get average of rgb values.
        avg = (cj[0] + cj[1] + cj[2]) / 3.0
        # cyan gets avg=0.6111, yellow gets avg=0.6337.
        # Choose cutoff for yellow and not cyan.
        if avg >= 0.625:
            text_color = 'black'
        else:
            text_color = 'white'
        text_colors.append(text_color)

    # Plot the array.
    img = pyplot.matshow(process_array, cmap=cmap, norm=norm, *args, **kwargs)

    # Add title and labels.
    if title is not None:
        pyplot.title(title)
    if xlabel is not None:
        pyplot.xlabel(xlabel)
    if ylabel is not None:
        pyplot.ylabel(ylabel)

    # Either invert y-axis, and put tick labels at bottom,
    # or put the x-axis label at the top.
    axis = pyplot.gca()
    if yflip:
        axis.invert_yaxis()
        for tick in axis.xaxis.iter_ticks():
            tick[0].label1On = True
            tick[0].label2On = False
    else:
        axis.xaxis.set_label_position('top')

    # Label each cell.
    for row in range(values_array.shape[0]):
        for col in range(values_array.shape[1]):
            process = process_array[row, col]
            value = values_array[row, col]
            label = '%d' % (value)
            color = text_colors[process]
            pyplot.text(
                col, row, label,
                horizontalalignment='center',
                verticalalignment='center',
                color=color)

    # Add colorbar legend.
    if legend:
        cbar = pyplot.colorbar(img)
        cbar_ticks = [0.5 + p for p in range(num_processors)]
        cbar_labels = [str(p) for p in range(num_processors)]
        cbar.set_ticks(cbar_ticks)
        cbar.set_ticklabels(cbar_labels)
        cbar.set_label('Processor')

    # Adjust size.
    figure = pyplot.gcf()
    figure.set_size_inches(10.0, 5.0)

    # Save to output file.
    if filename is not None:
        pyplot.savefig(filename, dpi=100)

    return process_darray
<|endoftext|>


<commit_before>from aleph.core import create_app, celery as app  # noqa

from aleph.ingest import ingest  # noqa
from aleph.index.documents import index_document_id  # noqa
from aleph.logic.entities import bulk_load_query  # noqa
from aleph.logic.collections import process_collection, delete_collection  # noqa
from aleph.logic.documents import process_document_id  # noqa
from aleph.logic.scheduled import background  # noqa
from aleph.logic.xref import xref_collection # noqa
from aleph.logic.alerts import check_alerts  # noqa

flask_app = create_app()
flask_app.app_context().push()
<commit_msg>Hotfix queue/.<commit_after>from aleph.core import create_app, celery as app  # noqa

from aleph.ingest import ingest  # noqa
from aleph.index.documents import index_document_id  # noqa
from aleph.logic.entities import bulk_load_query  # noqa
from aleph.logic.collections import process_collection, delete_collection  # noqa
from aleph.logic.documents import process_document_id  # noqa
from aleph.logic.scheduled import hourly, daily  # noqa
from aleph.logic.xref import xref_collection # noqa
from aleph.logic.alerts import check_alerts  # noqa

flask_app = create_app()
flask_app.app_context().push()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
----------------------------------------
Created on Thu Sep 22 15:08:36 2016

@author: Adam Ek
----------------------------------------

Random indexing:

    (1): Generate index vectors
        gather all the words in the data:
            for each word, generate a d-dimensional matrix of 0's with a small amount of (+1) and (-1) distributed

    (2):
        scan text
            each time a word w occurs in a context, add that contexts index vector to the vector of that word

    Init
        summ of vectors
        normalize vec

"""
import numpy as np
from numpy import linalg as la
import re
import random
import math
import sklearn.metrics.pairwise as d
import matplotlib.pyplot as plt
from scipy import linalg as sla
import sklearn.decomposition as dec

class distrib_semantics():

    def __init__(self):
        self.vocabulary = []
        self.i_vectors = []
        self.word_vectors = []
        self.superlist = []
        self.svded = []

        self.weights = []
        self.total_words = 0
        self.sentences_total = 0

    #create an index vector for each word
    def rand_index_vector(self):
        arr = np.zeros(1024)

        for i in range(0, 4):
            if i%2 == 0:
                arr[random.randint(0, 1023)] = 1
            else:
                arr[random.randint(0, 1023)] = -1

        return arr

    def vectorizer(self, formatted_sentence):
        added = []
        self.sentences_total += 1
        for i, word in enumerate(formatted_sentence):

            #remove special things inside words
            formatted_sentence[i] = re.sub('[^a-zåäö0-9%-]', '', formatted_sentence[i])
            word = re.sub('[^a-zåäö0-9%-]', '', word)

            #len(word) == 0:
            #dont add null words
            if word == '':
                continue #continue does not add ''

            self.total_words += 1
            ###### FINE TUNE DATA
            #change numbers to NUM
            if word.lstrip('-').isdigit():
                word = 'NUM'
                formatted_sentence[i] = 'NUM'
            ### NUMS -> '5-6' etc
            elif re.match('\d+-\d+', word) is not None:
                word = 'NUMS'
                formatted_sentence[i] = 'NUMS'
            #percentages -> 12% etc
            elif '%' in word:
                word = 'PERC'
                formatted_sentence[i] = 'PERC'

            #set up vectors
            if word not in self.vocabulary:
                self.vocabulary.append(word)
                self.i_vectors.append(self.rand_index_vector())
                self.word_vectors.append(np.zeros(1024))
                self.weights.append([0, 1, 1])
            #create weight tools
            else:
                self.weights[self.vocabulary.index(word)][1] += 1
                if word not in added:
                    self.weights[self.vocabulary.index(word)][2] += 1
                    added.append(word)

    def create_vectors(self, filenames):
        for filename in filenames:
            with open(filename) as training:
                for row in training:
                     #separate row into sentences
                     row = row.strip().split('. ')
                     for sentence in row:
                        sentence = sentence.split()
                        #format sentence
                        #remove special things before/after word
                        formatted_sentence = [w.strip().lower() for w in sentence]
                        #only add sentences longer than 0 words
                        if len(formatted_sentence) > 0:
                            #create index_vector for word
                            #currently takes a senttence, change so it takes a word???
                            self.vectorizer(formatted_sentence)

                            #remove [''] and '' elements
                            while '' in formatted_sentence:
                                formatted_sentence.remove('')

                            self.superlist.append(formatted_sentence)

        self.apply_weights()
        self.create_word_vectors()

    def apply_weights(self):
        ###### Differrent weights if word is behind/after???
        ##Create weights for indexes
        for i, w in enumerate(self.weights):
            #tf-idf weight
            idf = math.log(self.sentences_total/w[2])
            self.weights[i][0] = (w[1]/self.total_words)*idf

    def create_word_vectors(self):
        window = 1 #how many words before/after to consider being a part of the context
        #weight for sliding window > 1
        #create contexts
        for sentence in self.superlist:
            for i, word in enumerate(sentence):
                ###SLIDING WINDOW PARAMETER
                for n in range(1,window+1):
                    #words before
                    try:
                        if i - n >= 0: #exclude negative indexes
                            prev_word = self.vocabulary.index(sentence[i-n])
                            self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[prev_word] * self.weights[prev_word][0])
                    except:
                        pass

                    #words after
                    try:
                        next_word = self.vocabulary.index(sentence[i+n])
                        self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[next_word] * self.weights[next_word][0])
                    except:
                        pass
        #empty sentence list incase update command
        self.superlist = []

    #######################################################
    ################## EVUALUATIONS #######################
    #######################################################
    def find_similarity(self, word1, word2):

        #check if the words exists
        if word1 not in self.vocabulary:
            return ' '.join([word1, "does not exist, try again"])
        elif word2 not in self.vocabulary:
            return ' '.join([word2, "does not exist, try again"])
        else:
            i_word1 = self.word_vectors[self.vocabulary.index(word1)]
            i_word2 = self.word_vectors[self.vocabulary.index(word2)]

        ### Check definition and implementation
        cosine_sim = d.cosine_similarity(i_word1.reshape(1,-1), i_word2.reshape(1,-1))

        return(cosine_sim[0][0])

    #top 3
    def similarity_top(self, word):

        if word not in self.vocabulary:
            return ' '.join([word, "does not exist, try again"])
        else:
            ind = self.vocabulary.index(word)
            word = self.word_vectors[ind]

        top = [0,0,0]
        word_top = ["","",""]

        for i, vect in enumerate(self.word_vectors):
            if i == ind:
                continue

            cs = d.cosine_similarity(word.reshape(1,-1), vect.reshape(1,-1))

            if cs > top[0]:
                top[0] = cs
                word_top[0] = self.vocabulary[i]
            elif cs > top[1]:
                top[1] = cs
                word_top[1] = self.vocabulary[i]
            elif cs > top[2]:
                top[2] = cs
                word_top[2] = self.vocabulary[i]

        return [(x, y[0][0]) for x, y in zip(word_top,top)]

    def save(self):
        outfile1 = '/home/usr1/Python_Prg_1/SU_PY/project/np_1.npy'
        outfile2 = '/home/usr1/Python_Prg_1/SU_PY/project/np_2.npy'
        outfile3 = '/home/usr1/Python_Prg_1/SU_PY/project/np_3.npy'
        outfile4 = '/home/usr1/Python_Prg_1/SU_PY/project/np_4.npy'
        outfile5 = '/home/usr1/Python_Prg_1/SU_PY/project/np_5.npy'
        outfile6 = '/home/usr1/Python_Prg_1/SU_PY/project/np_6.npy'

        np.save(outfile1, self.vocabulary)
        np.save(outfile2, self.word_vectors)
        np.save(outfile3, self.i_vectors)
        np.save(outfile4, self.weights)
        np.save(outfile5, self.total_words)
        np.save(outfile6, self.sentences_total)

    def load(self):
        self.vocabulary = list(np.load('/home/usr1/Python_Prg_1/SU_PY/project/np_1.npy'))
        self.word_vectors = list(np.load('/home/usr1/Python_Prg_1/SU_PY/project/np_2.npy'))
        self.i_vectors = list(np.load('/home/usr1/Python_Prg_1/SU_PY/project/np_3.npy'))
        self.weights = list(np.load('/home/usr1/Python_Prg_1/SU_PY/project/np_4.npy'))
        self.total_words = np.load('/home/usr1/Python_Prg_1/SU_PY/project/np_5.npy')
        self.sentences_total = np.load('/home/usr1/Python_Prg_1/SU_PY/project/np_6.npy')

    def update(self, path):
        self.create_vectors([path])

    def info(self):
        print(len(self.vocabulary), 'word in vocabulary')
        print(self.sentences_total, 'total sentences')
        print(self.total_words, 'total words')

def main():
#    x = distrib_semantics(['/home/usr1/Python_Prg_1/SU_PY/project/gutenberg/austen-emma.txt'])
    x = distrib_semantics()

    #,'/home/usr1/PythonPrg/project/gutenberg/austen-emma.txt','/home/usr1/PythonPrg/project/gutenberg/austen-sense.txt'

    print("Welcome to Distributial Semantics with Random Indexing")

    print("Enter new data source by typing 'data', load by typing 'load'")
    setup = input('> ')
    setup = setup.split()
    if setup[0] == 'data':
        x.create_vectors(['/home/usr1/Python_Prg_1/SU_PY/project/test_doc_1.txt'])

    elif setup[0] == 'load':
        x.load()

    print("Type 'sim word1 word2' for similarity between two words, 'top word' for top 3 similar words and 'exit' to quit")
    while True:
        choice = input('> ')
        input_args = choice.split()

        #fix error output
        if input_args[0] == 'sim':
           sim = x.find_similarity(input_args[1].lower(), input_args[2].lower())
           print('Cosine similarity between', input_args[1], 'and',input_args[2] ,'is\n', sim, '\n')

        #fix error output
        elif input_args[0] == 'top':
           top = x.similarity_top(input_args[1].lower())
           print('Top similar words for', input_args[1], 'is:')
           for x, y in top:
               print(x, y)

        elif input_args[0] == 'exit':
           break

        elif input_args[0] == 'save':
            x.save()

        #fix error in path
        elif input_args[0] == 'update':
            x.update(input_args[1])

        elif input_args[0] == 'info':
            x.info()

        elif input_args[0] == 'help':
            print("'sim word1 word2' for similarity")
            print("'top word' for top 3 similar words")
            print("'save' to save current data")
            print("'update path' to update the data with a new textfile")
            print("'info' for info about the data")
            print("'exit' to quit")

        else:
           print("Unrecognized command")

        #warm-nominal
#    sim = x.find_similarity('language', 'the')
#    print(sim)
#    top = x.similarity_top('language')Women
#    print(top)

if __name__ == '__main__':
    main()

<commit_msg>saving, loading, user interface<commit_after># -*- coding: utf-8 -*-
"""
----------------------------------------
Created on Thu Sep 22 15:08:36 2016

@author: Adam Ek
----------------------------------------

Random indexing:

    (1): Generate index vectors
        gather all the words in the data:
            for each word, generate a d-dimensional matrix of 0's with a small amount of (+1) and (-1) distributed

    (2):
        scan text
            each time a word w occurs in a context, add that contexts index vector to the vector of that word

    Init
        summ of vectors
        normalize vec

"""
import numpy as np
import re
import random
import math
import sklearn.metrics.pairwise as d
import matplotlib.pyplot as plt
from scipy import linalg as sla
import sklearn.decomposition as dec

class distrib_semantics():

    def __init__(self):
        self.vocabulary = []
        self.i_vectors = []
        self.word_vectors = []
        self.superlist = []
        self.svded = []

        self.documents = 0
        self.weights = []
        self.total_words = 0
        self.sentences_total = 0

    #create an index vector for each word
    def rand_index_vector(self):
        arr = np.zeros(1024)

        for i in range(0, 4):
            if i%2 == 0:
                arr[random.randint(0, 1023)] = 1
            else:
                arr[random.randint(0, 1023)] = -1

        return arr

    def vectorizer(self, formatted_sentence):
        added = []
        self.sentences_total += 1
        for i, word in enumerate(formatted_sentence):

            #remove special things inside words
            formatted_sentence[i] = re.sub('[^a-zåäö0-9%-]', '', formatted_sentence[i])
            word = re.sub('[^a-zåäö0-9%-]', '', word)

            #len(word) == 0:
            #dont add null words
            if word == '':
                continue #continue does not add ''

            self.total_words += 1
            ###### FINE TUNE DATA
            #change numbers to NUM
            if word.lstrip('-').isdigit():
                word = 'NUM'
                formatted_sentence[i] = 'NUM'
            ### NUMS -> '5-6' etc
            elif re.match('\d+-\d+', word) is not None:
                word = 'NUMS'
                formatted_sentence[i] = 'NUMS'
            #percentages -> 12% etc
            elif '%' in word:
                word = 'PERC'
                formatted_sentence[i] = 'PERC'

            #set up vectors
            if word not in self.vocabulary:
                self.vocabulary.append(word)
                self.i_vectors.append(self.rand_index_vector())
                self.word_vectors.append(np.zeros(1024))
                self.weights.append([0, 1, 1, self.documents])
            #create weight tools
            else:
                self.weights[self.vocabulary.index(word)][1] += 1

                if self.documents in self.weights[self.vocabulary.index(word)][3:]:
                    pass
                else:
                    self.weights[self.vocabulary.index(word)].append(self.documents)

                if word not in added:
                    self.weights[self.vocabulary.index(word)][2] += 1
                    added.append(word)

    def create_vectors(self, filenames):
        for filename in filenames:
            try:
                with open(filename) as training:
                    self.documents += 1
                    for row in training:
                         #separate row into sentences
                         row = row.strip().split('. ')
                         for sentence in row:
                            sentence = sentence.split()
                            #format sentence
                            #remove special things before/after word
                            formatted_sentence = [w.strip().lower() for w in sentence]
                            #only add sentences longer than 0 words
                            if len(formatted_sentence) > 0:
                                #create index_vector for word
                                #currently takes a senttence, change so it takes a word???
                                self.vectorizer(formatted_sentence)

                                #remove [''] and '' elements
                                while '' in formatted_sentence:
                                    formatted_sentence.remove('')

                                self.superlist.append(formatted_sentence)

            except FileNotFoundError:
                print("File not found, try again")

        self.apply_weights()
        self.create_word_vectors()

    def apply_weights(self):
        ###### Differrent weights if word is behind/after???
        ##Create weights for indexes
        for i, w in enumerate(self.weights):
            #tf-idf weight
            #real idf
            idf1 = math.log(self.documents/len(self.weights[i][3:]))
            idf = math.log(self.sentences_total/w[2])
            self.weights[i][0] = (w[1]/self.total_words)*idf

    def create_word_vectors(self):
        window = 1 #how many words before/after to consider being a part of the context
        #weight for sliding window > 1
        #create contexts
        for sentence in self.superlist:
            for i, word in enumerate(sentence):
                ###SLIDING WINDOW PARAMETER
                for n in range(1,window+1):
                    #words before
                    try:
                        if i - n >= 0: #exclude negative indexes
                            prev_word = self.vocabulary.index(sentence[i-n])
                            self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[prev_word] * self.weights[prev_word][0])
                    except:
                        pass

                    #words after
                    try:
                        next_word = self.vocabulary.index(sentence[i+n])
                        self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[next_word] * self.weights[next_word][0])
                    except:
                        pass
        #empty sentence list incase update command
        self.superlist = []

    #######################################################
    ################## EVUALUATIONS #######################
    #######################################################
    def find_similarity(self, word1, word2):

        #check if the words exists
        if word1 not in self.vocabulary:
            return ' '.join([word1, "does not exist, try again"])
        elif word2 not in self.vocabulary:
            return ' '.join([word2, "does not exist, try again"])
        else:
            i_word1 = self.word_vectors[self.vocabulary.index(word1)]
            i_word2 = self.word_vectors[self.vocabulary.index(word2)]

        ### Check definition and implementation
        cosine_sim = d.cosine_similarity(i_word1.reshape(1,-1), i_word2.reshape(1,-1))

        return(cosine_sim[0][0])

    #top 3
    def similarity_top(self, word):

        if word not in self.vocabulary:
            return ' '.join([word, "does not exist, try again"])
        else:
            ind = self.vocabulary.index(word)
            word = self.word_vectors[ind]

        top = [0,0,0]
        word_top = ["","",""]

        for i, vect in enumerate(self.word_vectors):
            if i == ind:
                continue

            cs = d.cosine_similarity(word.reshape(1,-1), vect.reshape(1,-1))

            if cs > top[0]:
                top[0] = cs
                word_top[0] = self.vocabulary[i]
            elif cs > top[1]:
                top[1] = cs
                word_top[1] = self.vocabulary[i]
            elif cs > top[2]:
                top[2] = cs
                word_top[2] = self.vocabulary[i]

        return [(x, y[0][0]) for x, y in zip(word_top,top)]

    def save(self):
        outfile1 = '/home/usr1/git/dist_data/np_1.npy'
        outfile2 = '/home/usr1/git/dist_data/np_2.npy'
        outfile3 = '/home/usr1/git/dist_data/np_3.npy'
        outfile4 = '/home/usr1/git/dist_data/np_4.npy'
        outfile5 = '/home/usr1/git/dist_data/np_5.npy'
        outfile6 = '/home/usr1/git/dist_data/np_6.npy'
        outfile7 = '/home/usr1/git/dist_data/np_7.npy'

        np.save(outfile1, self.vocabulary)
        np.save(outfile2, self.word_vectors)
        np.save(outfile3, self.i_vectors)
        np.save(outfile4, self.weights)
        np.save(outfile5, self.total_words)
        np.save(outfile6, self.sentences_total)
        np.save(outfile7, self.documents)

    def load(self):
        self.vocabulary = list(np.load('/home/usr1/git/dist_data/np_1.npy'))
        self.word_vectors = list(np.load('/home/usr1/git/dist_data/np_2.npy'))
        self.i_vectors = list(np.load('/home/usr1/git/dist_data/np_3.npy'))
        self.weights = list(np.load('/home/usr1/git/dist_data/np_4.npy'))
        self.total_words = np.load('/home/usr1/git/dist_data/np_5.npy')
        self.sentences_total = np.load('/home/usr1/git/dist_data/np_6.npy')
        self.documents = np.load('/home/usr1/git/dist_data/np_7.npy')

    def update(self, path):
        self.create_vectors([path])

    def info(self, arg):

        if arg != None:
            if arg not in self.vocabulary:
                print(arg, 'does not exist')
            else:
                print(arg)
                print('occurences:', self.weights[self.vocabulary.index(arg)][1])
                print('in', self.weights[self.vocabulary.index(arg)][2], 'sentences')
                print('total weight of word:', self.weights[self.vocabulary.index(arg)][0])
                print('word in documents:', self.weights[self.vocabulary.index(arg)][3:])
                print('idf', math.log(self.documents/len(self.weights[self.vocabulary.index(arg)][3:])))

        else:

            print(len(self.vocabulary), 'unique words in vocabulary')
            print(self.sentences_total, 'total sentences')
            print(self.total_words, 'total words')
            print(self.documents, 'total documents')

def main():
#    x = distrib_semantics(['/home/usr1/Python_Prg_1/SU_PY/project/gutenberg/austen-emma.txt'])
    x = distrib_semantics()

    #,'/home/usr1/PythonPrg/project/gutenberg/austen-emma.txt','/home/usr1/PythonPrg/project/gutenberg/austen-sense.txt'

    print("Welcome to Distributial Semantics with Random Indexing")

    #load data
    while True:
        print("Enter new data source by typing 'new path', load by typing 'load'")
        setup = input('> ')
        setup = setup.split()
        if setup[0] == 'load':
            x.load()
            break
        elif setup[0] == 'new':
            x.create_vectors(['/home/usr1/PythonPrg/project/test_doc_1.txt'])
            break
        else:
            print('Invalid input')

    #User interface after data has loaded
    print("Type 'sim word1 word2' for similarity between two words, 'top word' for top 3 similar words and 'exit' to quit")
    while True:
        choice = input('> ')
        input_args = choice.split()

        if input_args == '':
            print('Please try again')
        #fix error output
        elif input_args[0] == 'sim':
            try:
                sim = x.find_similarity(input_args[1].lower(), input_args[2].lower())
                if sim == str(sim):
                    print(sim)
                else:
                    print('Cosine similarity between', input_args[1], 'and',input_args[2] ,'is\n', sim, '\n')
            except:
                print('Invalid input')
        #fix error output
        elif input_args[0] == 'top':
            try:
                top = x.similarity_top(input_args[1].lower())
                print('Top similar words for', input_args[1], 'is:')
                for x, y in top:
                    print(x, y)
            except:
                print('Invalid input')

        elif input_args[0] == 'exit':
           break

        elif input_args[0] == 'save':
            x.save()

        elif input_args[0] == 'update':
            try:
                x.update(input_args[1])
            except:
                print('Please provide a path')

        elif input_args[0] == 'info':
            try:
                x.info(input_args[1].lower())
            except:
                x.info(None)

        elif input_args[0] == 'help':
            print("'sim word1 word2' for similarity")
            print("'top word' for top 3 similar words")
            print("'save' to save current data")
            print("'update path' to update the data with a new textfile")
            print("'info' for info about the data")
            print("'exit' to quit")

        else:
           print("Unrecognized command")

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

def sum_list(num_ls):
    """Sum number list by recursion."""
    if len(num_ls) == 1:
    	return num_ls[0]
    else:
    	return num_ls[0] + sum_list(num_ls[1:])

def main():
	num_ls = [0, 1, 2, 3, 4, 5]
	print('Sum of {}: {}'.format(num_ls, sum_list(num_ls)))

if __name__ == '__main__':
	main()
<commit_msg>Complete benchmarking: for vs. recur<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

def sum_list_for(num_ls):
    """Sum number list by for loop."""
    _sum = 0
    for num in num_ls:
        _sum += num
    return _sum

def sum_list_recur(num_ls):
    """Sum number list by recursion."""
    if len(num_ls) == 1:
        return num_ls[0]
    else:
        return num_ls[0] + sum_list_recur(num_ls[1:])

def main():
    import time

    num_ls = [0, 1, 2, 3, 4, 5]

    start_time = time.time()
    print('By for loop: {}'.format(sum_list_for(num_ls)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By recursion: {}'.format(sum_list_recur(num_ls)))
    print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals

# About name_local: capitalize it as if your language name was appearing
# inside a sentence in your language.

LANG_INFO = {
    'ar': {
        'bidi': True,
        'code': 'ar',
        'name': 'Arabic',
        'name_local': '\u0627\u0644\u0639\u0631\u0628\u064a\u0651\u0629',
    },
    'az': {
        'bidi': True,
        'code': 'az',
        'name': 'Azerbaijani',
        'name_local': 'az\u0259rbaycan dili',
    },
    'bg': {
        'bidi': False,
        'code': 'bg',
        'name': 'Bulgarian',
        'name_local': '\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438',
    },
    'bn': {
        'bidi': False,
        'code': 'bn',
        'name': 'Bengali',
        'name_local': '\u09ac\u09be\u0982\u09b2\u09be',
    },
    'bs': {
        'bidi': False,
        'code': 'bs',
        'name': 'Bosnian',
        'name_local': 'bosanski',
    },
    'ca': {
        'bidi': False,
        'code': 'ca',
        'name': 'Catalan',
        'name_local': 'catal\xe0',
    },
    'cs': {
        'bidi': False,
        'code': 'cs',
        'name': 'Czech',
        'name_local': '\u010desky',
    },
    'cy': {
        'bidi': False,
        'code': 'cy',
        'name': 'Welsh',
        'name_local': 'Cymraeg',
    },
    'da': {
        'bidi': False,
        'code': 'da',
        'name': 'Danish',
        'name_local': 'Dansk',
    },
    'de': {
        'bidi': False,
        'code': 'de',
        'name': 'German',
        'name_local': 'Deutsch',
    },
    'el': {
        'bidi': False,
        'code': 'el',
        'name': 'Greek',
        'name_local': '\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac',
    },
    'en': {
        'bidi': False,
        'code': 'en',
        'name': 'English',
        'name_local': 'English',
    },
    'en-gb': {
        'bidi': False,
        'code': 'en-gb',
        'name': 'British English',
        'name_local': 'British English',
    },
    'eo': {
        'bidi': False,
        'code': 'eo',
        'name': 'Esperanto',
        'name_local': 'Esperanto',
    },
    'es': {
        'bidi': False,
        'code': 'es',
        'name': 'Spanish',
        'name_local': 'espa\xf1ol',
    },
    'es-ar': {
        'bidi': False,
        'code': 'es-ar',
        'name': 'Argentinian Spanish',
        'name_local': 'espa\xf1ol de Argentina',
    },
    'es-mx': {
        'bidi': False,
        'code': 'es-mx',
        'name': 'Mexican Spanish',
        'name_local': 'espa\xf1ol de Mexico',
    },
    'es-ni': {
        'bidi': False,
        'code': 'es-ni',
        'name': 'Nicaraguan Spanish',
        'name_local': 'espa\xf1ol de Nicaragua',
    },
    'et': {
        'bidi': False,
        'code': 'et',
        'name': 'Estonian',
        'name_local': 'eesti',
    },
    'eu': {
        'bidi': False,
        'code': 'eu',
        'name': 'Basque',
        'name_local': 'Basque',
    },
    'fa': {
        'bidi': True,
        'code': 'fa',
        'name': 'Persian',
        'name_local': '\u0641\u0627\u0631\u0633\u06cc',
    },
    'fi': {
        'bidi': False,
        'code': 'fi',
        'name': 'Finnish',
        'name_local': 'suomi',
    },
    'fr': {
        'bidi': False,
        'code': 'fr',
        'name': 'French',
        'name_local': 'fran\xe7ais',
    },
    'fy-nl': {
        'bidi': False,
        'code': 'fy-nl',
        'name': 'Frisian',
        'name_local': 'Frisian',
    },
    'ga': {
        'bidi': False,
        'code': 'ga',
        'name': 'Irish',
        'name_local': 'Gaeilge',
    },
    'gl': {
        'bidi': False,
        'code': 'gl',
        'name': 'Galician',
        'name_local': 'galego',
    },
    'he': {
        'bidi': True,
        'code': 'he',
        'name': 'Hebrew',
        'name_local': '\u05e2\u05d1\u05e8\u05d9\u05ea',
    },
    'hi': {
        'bidi': False,
        'code': 'hi',
        'name': 'Hindi',
        'name_local': 'Hindi',
    },
    'hr': {
        'bidi': False,
        'code': 'hr',
        'name': 'Croatian',
        'name_local': 'Hrvatski',
    },
    'hu': {
        'bidi': False,
        'code': 'hu',
        'name': 'Hungarian',
        'name_local': 'Magyar',
    },
    'id': {
        'bidi': False,
        'code': 'id',
        'name': 'Indonesian',
        'name_local': 'Bahasa Indonesia',
    },
    'is': {
        'bidi': False,
        'code': 'is',
        'name': 'Icelandic',
        'name_local': '\xcdslenska',
    },
    'it': {
        'bidi': False,
        'code': 'it',
        'name': 'Italian',
        'name_local': 'italiano',
    },
    'ja': {
        'bidi': False,
        'code': 'ja',
        'name': 'Japanese',
        'name_local': '\u65e5\u672c\u8a9e',
    },
    'ka': {
        'bidi': False,
        'code': 'ka',
        'name': 'Georgian',
        'name_local': '\u10e5\u10d0\u10e0\u10d7\u10e3\u10da\u10d8',
    },
    'kk': {
        'bidi': False,
        'code': 'kk',
        'name': 'Kazakh',
        'name_local': '\u049a\u0430\u0437\u0430\u049b',
    },
    'km': {
        'bidi': False,
        'code': 'km',
        'name': 'Khmer',
        'name_local': 'Khmer',
    },
    'kn': {
        'bidi': False,
        'code': 'kn',
        'name': 'Kannada',
        'name_local': 'Kannada',
    },
    'ko': {
        'bidi': False,
        'code': 'ko',
        'name': 'Korean',
        'name_local': '\ud55c\uad6d\uc5b4',
    },
    'lt': {
        'bidi': False,
        'code': 'lt',
        'name': 'Lithuanian',
        'name_local': 'Lithuanian',
    },
    'lv': {
        'bidi': False,
        'code': 'lv',
        'name': 'Latvian',
        'name_local': 'latvie\u0161u',
    },
    'mk': {
        'bidi': False,
        'code': 'mk',
        'name': 'Macedonian',
        'name_local': '\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438',
    },
    'ml': {
        'bidi': False,
        'code': 'ml',
        'name': 'Malayalam',
        'name_local': 'Malayalam',
    },
    'mn': {
        'bidi': False,
        'code': 'mn',
        'name': 'Mongolian',
        'name_local': 'Mongolian',
    },
    'nb': {
        'bidi': False,
        'code': 'nb',
        'name': 'Norwegian Bokmal',
        'name_local': 'Norsk (bokm\xe5l)',
    },
    'ne': {
        'bidi': False,
        'code': 'ne',
        'name': 'Nepali',
        'name_local': '\u0928\u0947\u092a\u093e\u0932\u0940',
    },
    'nl': {
        'bidi': False,
        'code': 'nl',
        'name': 'Dutch',
        'name_local': 'Nederlands',
    },
    'nn': {
        'bidi': False,
        'code': 'nn',
        'name': 'Norwegian Nynorsk',
        'name_local': 'Norsk (nynorsk)',
    },
    'no': {
        'bidi': False,
        'code': 'no',
        'name': 'Norwegian',
        'name_local': 'Norsk',
    },
    'pa': {
        'bidi': False,
        'code': 'pa',
        'name': 'Punjabi',
        'name_local': 'Punjabi',
    },
    'pl': {
        'bidi': False,
        'code': 'pl',
        'name': 'Polish',
        'name_local': 'polski',
    },
    'pt': {
        'bidi': False,
        'code': 'pt',
        'name': 'Portuguese',
        'name_local': 'Portugu\xeas',
    },
    'pt-br': {
        'bidi': False,
        'code': 'pt-br',
        'name': 'Brazilian Portuguese',
        'name_local': 'Portugu\xeas Brasileiro',
    },
    'ro': {
        'bidi': False,
        'code': 'ro',
        'name': 'Romanian',
        'name_local': 'Rom\xe2n\u0103',
    },
    'ru': {
        'bidi': False,
        'code': 'ru',
        'name': 'Russian',
        'name_local': '\u0420\u0443\u0441\u0441\u043a\u0438\u0439',
    },
    'sk': {
        'bidi': False,
        'code': 'sk',
        'name': 'Slovak',
        'name_local': 'slovensk\xfd',
    },
    'sl': {
        'bidi': False,
        'code': 'sl',
        'name': 'Slovenian',
        'name_local': 'Sloven\u0161\u010dina',
    },
    'sq': {
        'bidi': False,
        'code': 'sq',
        'name': 'Albanian',
        'name_local': 'Albanian',
    },
    'sr': {
        'bidi': False,
        'code': 'sr',
        'name': 'Serbian',
        'name_local': '\u0441\u0440\u043f\u0441\u043a\u0438',
    },
    'sr-latn': {
        'bidi': False,
        'code': 'sr-latn',
        'name': 'Serbian Latin',
        'name_local': 'srpski (latinica)',
    },
    'sv': {
        'bidi': False,
        'code': 'sv',
        'name': 'Swedish',
        'name_local': 'Svenska',
    },
    'sw': {
        'bidi': False,
        'code': 'sw',
        'name': 'Swahili',
        'name_local': 'Kiswahili',
    },
    'ta': {
        'bidi': False,
        'code': 'ta',
        'name': 'Tamil',
        'name_local': '\u0ba4\u0bae\u0bbf\u0bb4\u0bcd',
    },
    'te': {
        'bidi': False,
        'code': 'te',
        'name': 'Telugu',
        'name_local': '\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41',
    },
    'th': {
        'bidi': False,
        'code': 'th',
        'name': 'Thai',
        'name_local': 'Thai',
    },
    'tr': {
        'bidi': False,
        'code': 'tr',
        'name': 'Turkish',
        'name_local': 'T\xfcrk\xe7e',
    },
    'tt': {
        'bidi': False,
        'code': 'tt',
        'name': 'Tatar',
        'name_local': '\u0422\u0430\u0442\u0430\u0440\u0447\u0430',
    },
    'uk': {
        'bidi': False,
        'code': 'uk',
        'name': 'Ukrainian',
        'name_local': '\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430',
    },
    'ur': {
        'bidi': False,
        'code': 'ur',
        'name': 'Urdu',
        'name_local': '\u0627\u0631\u062f\u0648',
    },
    'vi': {
        'bidi': False,
        'code': 'vi',
        'name': 'Vietnamese',
        'name_local': 'Vietnamese',
    },
    'zh-cn': {
        'bidi': False,
        'code': 'zh-cn',
        'name': 'Simplified Chinese',
        'name_local': '\u7b80\u4f53\u4e2d\u6587',
    },
    'zh-tw': {
        'bidi': False,
        'code': 'zh-tw',
        'name': 'Traditional Chinese',
        'name_local': '\u7e41\u9ad4\u4e2d\u6587',
    }
}
<commit_msg>[1.5.x] Fixed #19174 -- Fixed capitalization errors in LANG_INFO<commit_after>from __future__ import unicode_literals

# About name_local: capitalize it as if your language name was appearing
# inside a sentence in your language.

LANG_INFO = {
    'ar': {
        'bidi': True,
        'code': 'ar',
        'name': 'Arabic',
        'name_local': '\u0627\u0644\u0639\u0631\u0628\u064a\u0651\u0629',
    },
    'az': {
        'bidi': True,
        'code': 'az',
        'name': 'Azerbaijani',
        'name_local': 'az\u0259rbaycan dili',
    },
    'bg': {
        'bidi': False,
        'code': 'bg',
        'name': 'Bulgarian',
        'name_local': '\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438',
    },
    'bn': {
        'bidi': False,
        'code': 'bn',
        'name': 'Bengali',
        'name_local': '\u09ac\u09be\u0982\u09b2\u09be',
    },
    'bs': {
        'bidi': False,
        'code': 'bs',
        'name': 'Bosnian',
        'name_local': 'bosanski',
    },
    'ca': {
        'bidi': False,
        'code': 'ca',
        'name': 'Catalan',
        'name_local': 'catal\xe0',
    },
    'cs': {
        'bidi': False,
        'code': 'cs',
        'name': 'Czech',
        'name_local': '\u010desky',
    },
    'cy': {
        'bidi': False,
        'code': 'cy',
        'name': 'Welsh',
        'name_local': 'Cymraeg',
    },
    'da': {
        'bidi': False,
        'code': 'da',
        'name': 'Danish',
        'name_local': 'dansk',
    },
    'de': {
        'bidi': False,
        'code': 'de',
        'name': 'German',
        'name_local': 'Deutsch',
    },
    'el': {
        'bidi': False,
        'code': 'el',
        'name': 'Greek',
        'name_local': '\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac',
    },
    'en': {
        'bidi': False,
        'code': 'en',
        'name': 'English',
        'name_local': 'English',
    },
    'en-gb': {
        'bidi': False,
        'code': 'en-gb',
        'name': 'British English',
        'name_local': 'British English',
    },
    'eo': {
        'bidi': False,
        'code': 'eo',
        'name': 'Esperanto',
        'name_local': 'Esperanto',
    },
    'es': {
        'bidi': False,
        'code': 'es',
        'name': 'Spanish',
        'name_local': 'espa\xf1ol',
    },
    'es-ar': {
        'bidi': False,
        'code': 'es-ar',
        'name': 'Argentinian Spanish',
        'name_local': 'espa\xf1ol de Argentina',
    },
    'es-mx': {
        'bidi': False,
        'code': 'es-mx',
        'name': 'Mexican Spanish',
        'name_local': 'espa\xf1ol de Mexico',
    },
    'es-ni': {
        'bidi': False,
        'code': 'es-ni',
        'name': 'Nicaraguan Spanish',
        'name_local': 'espa\xf1ol de Nicaragua',
    },
    'et': {
        'bidi': False,
        'code': 'et',
        'name': 'Estonian',
        'name_local': 'eesti',
    },
    'eu': {
        'bidi': False,
        'code': 'eu',
        'name': 'Basque',
        'name_local': 'Basque',
    },
    'fa': {
        'bidi': True,
        'code': 'fa',
        'name': 'Persian',
        'name_local': '\u0641\u0627\u0631\u0633\u06cc',
    },
    'fi': {
        'bidi': False,
        'code': 'fi',
        'name': 'Finnish',
        'name_local': 'suomi',
    },
    'fr': {
        'bidi': False,
        'code': 'fr',
        'name': 'French',
        'name_local': 'fran\xe7ais',
    },
    'fy-nl': {
        'bidi': False,
        'code': 'fy-nl',
        'name': 'Frisian',
        'name_local': 'Frisian',
    },
    'ga': {
        'bidi': False,
        'code': 'ga',
        'name': 'Irish',
        'name_local': 'Gaeilge',
    },
    'gl': {
        'bidi': False,
        'code': 'gl',
        'name': 'Galician',
        'name_local': 'galego',
    },
    'he': {
        'bidi': True,
        'code': 'he',
        'name': 'Hebrew',
        'name_local': '\u05e2\u05d1\u05e8\u05d9\u05ea',
    },
    'hi': {
        'bidi': False,
        'code': 'hi',
        'name': 'Hindi',
        'name_local': 'Hindi',
    },
    'hr': {
        'bidi': False,
        'code': 'hr',
        'name': 'Croatian',
        'name_local': 'Hrvatski',
    },
    'hu': {
        'bidi': False,
        'code': 'hu',
        'name': 'Hungarian',
        'name_local': 'Magyar',
    },
    'id': {
        'bidi': False,
        'code': 'id',
        'name': 'Indonesian',
        'name_local': 'Bahasa Indonesia',
    },
    'is': {
        'bidi': False,
        'code': 'is',
        'name': 'Icelandic',
        'name_local': '\xcdslenska',
    },
    'it': {
        'bidi': False,
        'code': 'it',
        'name': 'Italian',
        'name_local': 'italiano',
    },
    'ja': {
        'bidi': False,
        'code': 'ja',
        'name': 'Japanese',
        'name_local': '\u65e5\u672c\u8a9e',
    },
    'ka': {
        'bidi': False,
        'code': 'ka',
        'name': 'Georgian',
        'name_local': '\u10e5\u10d0\u10e0\u10d7\u10e3\u10da\u10d8',
    },
    'kk': {
        'bidi': False,
        'code': 'kk',
        'name': 'Kazakh',
        'name_local': '\u049a\u0430\u0437\u0430\u049b',
    },
    'km': {
        'bidi': False,
        'code': 'km',
        'name': 'Khmer',
        'name_local': 'Khmer',
    },
    'kn': {
        'bidi': False,
        'code': 'kn',
        'name': 'Kannada',
        'name_local': 'Kannada',
    },
    'ko': {
        'bidi': False,
        'code': 'ko',
        'name': 'Korean',
        'name_local': '\ud55c\uad6d\uc5b4',
    },
    'lt': {
        'bidi': False,
        'code': 'lt',
        'name': 'Lithuanian',
        'name_local': 'Lithuanian',
    },
    'lv': {
        'bidi': False,
        'code': 'lv',
        'name': 'Latvian',
        'name_local': 'latvie\u0161u',
    },
    'mk': {
        'bidi': False,
        'code': 'mk',
        'name': 'Macedonian',
        'name_local': '\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438',
    },
    'ml': {
        'bidi': False,
        'code': 'ml',
        'name': 'Malayalam',
        'name_local': 'Malayalam',
    },
    'mn': {
        'bidi': False,
        'code': 'mn',
        'name': 'Mongolian',
        'name_local': 'Mongolian',
    },
    'nb': {
        'bidi': False,
        'code': 'nb',
        'name': 'Norwegian Bokmal',
        'name_local': 'norsk (bokm\xe5l)',
    },
    'ne': {
        'bidi': False,
        'code': 'ne',
        'name': 'Nepali',
        'name_local': '\u0928\u0947\u092a\u093e\u0932\u0940',
    },
    'nl': {
        'bidi': False,
        'code': 'nl',
        'name': 'Dutch',
        'name_local': 'Nederlands',
    },
    'nn': {
        'bidi': False,
        'code': 'nn',
        'name': 'Norwegian Nynorsk',
        'name_local': 'norsk (nynorsk)',
    },
    'no': {
        'bidi': False,
        'code': 'no',
        'name': 'Norwegian',
        'name_local': 'norsk',
    },
    'pa': {
        'bidi': False,
        'code': 'pa',
        'name': 'Punjabi',
        'name_local': 'Punjabi',
    },
    'pl': {
        'bidi': False,
        'code': 'pl',
        'name': 'Polish',
        'name_local': 'polski',
    },
    'pt': {
        'bidi': False,
        'code': 'pt',
        'name': 'Portuguese',
        'name_local': 'Portugu\xeas',
    },
    'pt-br': {
        'bidi': False,
        'code': 'pt-br',
        'name': 'Brazilian Portuguese',
        'name_local': 'Portugu\xeas Brasileiro',
    },
    'ro': {
        'bidi': False,
        'code': 'ro',
        'name': 'Romanian',
        'name_local': 'Rom\xe2n\u0103',
    },
    'ru': {
        'bidi': False,
        'code': 'ru',
        'name': 'Russian',
        'name_local': '\u0420\u0443\u0441\u0441\u043a\u0438\u0439',
    },
    'sk': {
        'bidi': False,
        'code': 'sk',
        'name': 'Slovak',
        'name_local': 'slovensk\xfd',
    },
    'sl': {
        'bidi': False,
        'code': 'sl',
        'name': 'Slovenian',
        'name_local': 'Sloven\u0161\u010dina',
    },
    'sq': {
        'bidi': False,
        'code': 'sq',
        'name': 'Albanian',
        'name_local': 'Albanian',
    },
    'sr': {
        'bidi': False,
        'code': 'sr',
        'name': 'Serbian',
        'name_local': '\u0441\u0440\u043f\u0441\u043a\u0438',
    },
    'sr-latn': {
        'bidi': False,
        'code': 'sr-latn',
        'name': 'Serbian Latin',
        'name_local': 'srpski (latinica)',
    },
    'sv': {
        'bidi': False,
        'code': 'sv',
        'name': 'Swedish',
        'name_local': 'svenska',
    },
    'sw': {
        'bidi': False,
        'code': 'sw',
        'name': 'Swahili',
        'name_local': 'Kiswahili',
    },
    'ta': {
        'bidi': False,
        'code': 'ta',
        'name': 'Tamil',
        'name_local': '\u0ba4\u0bae\u0bbf\u0bb4\u0bcd',
    },
    'te': {
        'bidi': False,
        'code': 'te',
        'name': 'Telugu',
        'name_local': '\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41',
    },
    'th': {
        'bidi': False,
        'code': 'th',
        'name': 'Thai',
        'name_local': 'Thai',
    },
    'tr': {
        'bidi': False,
        'code': 'tr',
        'name': 'Turkish',
        'name_local': 'T\xfcrk\xe7e',
    },
    'tt': {
        'bidi': False,
        'code': 'tt',
        'name': 'Tatar',
        'name_local': '\u0422\u0430\u0442\u0430\u0440\u0447\u0430',
    },
    'uk': {
        'bidi': False,
        'code': 'uk',
        'name': 'Ukrainian',
        'name_local': '\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430',
    },
    'ur': {
        'bidi': False,
        'code': 'ur',
        'name': 'Urdu',
        'name_local': '\u0627\u0631\u062f\u0648',
    },
    'vi': {
        'bidi': False,
        'code': 'vi',
        'name': 'Vietnamese',
        'name_local': 'Vietnamese',
    },
    'zh-cn': {
        'bidi': False,
        'code': 'zh-cn',
        'name': 'Simplified Chinese',
        'name_local': '\u7b80\u4f53\u4e2d\u6587',
    },
    'zh-tw': {
        'bidi': False,
        'code': 'zh-tw',
        'name': 'Traditional Chinese',
        'name_local': '\u7e41\u9ad4\u4e2d\u6587',
    }
}
<|endoftext|>