<commit_before># Copyright 2018 John Reese
# Licensed under the MIT license

"""
Core implementation of aiosqlite proxies
"""

import asyncio
import logging
import sqlite3
import sys
import warnings
from functools import partial
from pathlib import Path
from queue import Empty, Queue
from threading import Thread
from typing import (
    Any,
    AsyncIterator,
    Callable,
    Generator,
    Iterable,
    Optional,
    Type,
    Union,
)
from warnings import warn

from .context import contextmanager
from .cursor import Cursor

__all__ = ["connect", "Connection", "Cursor"]

LOG = logging.getLogger("aiosqlite")

def get_loop(future: asyncio.Future) -> asyncio.AbstractEventLoop:
    if sys.version_info >= (3, 7):
        return future.get_loop()
    else:
        return future._loop

class Connection(Thread):
    def __init__(
        self,
        connector: Callable[[], sqlite3.Connection],
        iter_chunk_size: int,
        loop: Optional[asyncio.AbstractEventLoop] = None,
    ) -> None:
        super().__init__()
        self._running = True
        self._connection: Optional[sqlite3.Connection] = None
        self._connector = connector
        self._tx: Queue = Queue()
        self._iter_chunk_size = iter_chunk_size

        if loop is not None:
            warn(
                "aiosqlite.Connection no longer uses the `loop` parameter",
                DeprecationWarning,
            )

    @property
    def _conn(self) -> sqlite3.Connection:
        if self._connection is None:
            raise ValueError("no active connection")

        return self._connection

    def _execute_insert(
        self, sql: str, parameters: Iterable[Any]
    ) -> Optional[sqlite3.Row]:
        cursor = self._conn.execute(sql, parameters)
        cursor.execute("SELECT last_insert_rowid()")
        return cursor.fetchone()

    def _execute_fetchall(
        self, sql: str, parameters: Iterable[Any]
    ) -> Iterable[sqlite3.Row]:
        cursor = self._conn.execute(sql, parameters)
        return cursor.fetchall()

    def run(self) -> None:
        """
        Execute function calls on a separate thread.

        :meta private:
        """
        while self._running:
            try:
                future, function = self._tx.get(timeout=0.1)
            except Empty:
                continue

            try:
                LOG.debug("executing %s", function)
                result = function()
                LOG.debug("returning %s", result)

                def set_result(fut, result):
                    if not fut.done():
                        fut.set_result(result)

                get_loop(future).call_soon_threadsafe(set_result, future, result)
            except BaseException as e:
                LOG.info("returning exception %s", e)

                def set_exception(fut, e):
                    if not fut.done():
                        fut.set_exception(e)

                get_loop(future).call_soon_threadsafe(set_exception, future, e)

    async def _execute(self, fn, *args, **kwargs):
        """Queue a function with the given arguments for execution."""
        function = partial(fn, *args, **kwargs)
        future = asyncio.get_event_loop().create_future()

        self._tx.put_nowait((future, function))

        return await future

    async def _connect(self) -> "Connection":
        """Connect to the actual sqlite database."""
        if self._connection is None:
            try:
                self._connection = await self._execute(self._connector)
            except Exception:
                self._running = False
                self._connection = None
                raise

        return self

    def __await__(self) -> Generator[Any, None, "Connection"]:
        self.start()
        return self._connect().__await__()

    async def __aenter__(self) -> "Connection":
        return await self

    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
        await self.close()

    @contextmanager
    async def cursor(self) -> Cursor:
        """Create an aiosqlite cursor wrapping a sqlite3 cursor object."""
        return Cursor(self, await self._execute(self._conn.cursor))

    async def commit(self) -> None:
        """Commit the current transaction."""
        await self._execute(self._conn.commit)

    async def rollback(self) -> None:
        """Roll back the current transaction."""
        await self._execute(self._conn.rollback)

    async def close(self) -> None:
        """Complete queued queries/cursors and close the connection."""
        try:
            await self._execute(self._conn.close)
        except Exception:
            LOG.info("exception occurred while closing connection")
            raise
        finally:
            self._running = False
            self._connection = None

    @contextmanager
    async def execute(self, sql: str, parameters: Iterable[Any] = None) -> Cursor:
        """Helper to create a cursor and execute the given query."""
        if parameters is None:
            parameters = []
        cursor = await self._execute(self._conn.execute, sql, parameters)
        return Cursor(self, cursor)

    @contextmanager
    async def execute_insert(
        self, sql: str, parameters: Iterable[Any] = None
    ) -> Optional[sqlite3.Row]:
        """Helper to insert and get the last_insert_rowid."""
        if parameters is None:
            parameters = []
        return await self._execute(self._execute_insert, sql, parameters)

    @contextmanager
    async def execute_fetchall(
        self, sql: str, parameters: Iterable[Any] = None
    ) -> Iterable[sqlite3.Row]:
        """Helper to execute a query and return all the data."""
        if parameters is None:
            parameters = []
        return await self._execute(self._execute_fetchall, sql, parameters)

    @contextmanager
    async def executemany(
        self, sql: str, parameters: Iterable[Iterable[Any]]
    ) -> Cursor:
        """Helper to create a cursor and execute the given multiquery."""
        cursor = await self._execute(self._conn.executemany, sql, parameters)
        return Cursor(self, cursor)

    @contextmanager
    async def executescript(self, sql_script: str) -> Cursor:
        """Helper to create a cursor and execute a user script."""
        cursor = await self._execute(self._conn.executescript, sql_script)
        return Cursor(self, cursor)

    async def interrupt(self) -> None:
        """Interrupt pending queries."""
        return self._conn.interrupt()

    async def create_function(
        self, name: str, num_params: int, func: Callable, deterministic: bool = False
    ) -> None:
        """
        Create user-defined function that can be later used
        within SQL statements. Must be run within the same thread
        that query executions take place so instead of executing directly
        against the connection, we defer this to `run` function.

        In Python 3.8 and above, if *deterministic* is true, the created
        function is marked as deterministic, which allows SQLite to perform
        additional optimizations. This flag is supported by SQLite 3.8.3 or
        higher, ``NotSupportedError`` will be raised if used with older
        versions.
        """
        if sys.version_info >= (3, 8):
            await self._execute(
                self._conn.create_function,
                name,
                num_params,
                func,
                deterministic=deterministic,
            )
        else:
            if deterministic:
                warnings.warn(
                    "Deterministic function support is only available on "
                    'Python 3.8+. Function "{}" will be registered as '
                    "non-deterministic as per SQLite defaults.".format(name)
                )

            await self._execute(self._conn.create_function, name, num_params, func)

    @property
    def in_transaction(self) -> bool:
        return self._conn.in_transaction

    @property
    def isolation_level(self) -> str:
        return self._conn.isolation_level

    @isolation_level.setter
    def isolation_level(self, value: str) -> None:
        self._conn.isolation_level = value

    @property
    def row_factory(self) -> "Optional[Type]":  # py3.5.2 compat (#24)
        return self._conn.row_factory

    @row_factory.setter
    def row_factory(self, factory: "Optional[Type]") -> None:  # py3.5.2 compat (#24)
        self._conn.row_factory = factory

    @property
    def text_factory(self) -> Type:
        return self._conn.text_factory

    @text_factory.setter
    def text_factory(self, factory: Type) -> None:
        self._conn.text_factory = factory

    @property
    def total_changes(self) -> int:
        return self._conn.total_changes

    async def enable_load_extension(self, value: bool) -> None:
        await self._execute(self._conn.enable_load_extension, value)  # type: ignore

    async def load_extension(self, path: str):
        await self._execute(self._conn.load_extension, path)  # type: ignore

    async def set_progress_handler(
        self, handler: Callable[[], Optional[int]], n: int
    ) -> None:
        await self._execute(self._conn.set_progress_handler, handler, n)

    async def set_trace_callback(self, handler: Callable) -> None:
        await self._execute(self._conn.set_trace_callback, handler)

    async def iterdump(self) -> AsyncIterator[str]:
        """
        Return an async iterator to dump the database in SQL text format.

        Example::

            async for line in db.iterdump():
                ...

        """
        dump_queue: Queue = Queue()

        def dumper():
            try:
                for line in self._conn.iterdump():
                    dump_queue.put_nowait(line)
                dump_queue.put_nowait(None)

            except Exception:
                LOG.exception("exception while dumping db")
                dump_queue.put_nowait(None)
                raise

        fut = self._execute(dumper)
        task = asyncio.ensure_future(fut)

        while True:
            try:
                line: Optional[str] = dump_queue.get_nowait()
                if line is None:
                    break
                yield line

            except Empty:
                if task.done():
                    LOG.warning("iterdump completed unexpectedly")
                    break

                await asyncio.sleep(0.01)

        await task

    async def backup(
        self,
        target: Union["Connection", sqlite3.Connection],
        *,
        pages: int = 0,
        progress: Optional[Callable[[int, int, int], None]] = None,
        name: str = "main",
        sleep: float = 0.250
    ) -> None:
        """
        Make a backup of the current database to the target database.

        Takes either a standard sqlite3 or aiosqlite Connection object as the target.
        """
        if sys.version_info < (3, 7):
            raise RuntimeError("backup() method is only available on Python 3.7+")

        if isinstance(target, Connection):
            target = target._conn

        await self._execute(
            self._conn.backup,
            target,
            pages=pages,
            progress=progress,
            name=name,
            sleep=sleep,
        )

def connect(
    database: Union[str, Path],
    *,
    iter_chunk_size=64,
    loop: Optional[asyncio.AbstractEventLoop] = None,
    **kwargs: Any
) -> Connection:
    """Create and return a connection proxy to the sqlite database."""

    if loop is not None:
        warn(
            "aiosqlite.connect() no longer uses the `loop` parameter",
            DeprecationWarning,
        )

    def connector() -> sqlite3.Connection:
        if isinstance(database, str):
            loc = database
        elif isinstance(database, bytes):
            loc = database.decode("utf-8")
        else:
            loc = str(database)

        return sqlite3.connect(loc, **kwargs)

    return Connection(connector, iter_chunk_size)
<commit_msg>Reduce exception logging severity<commit_after># Copyright 2018 John Reese
# Licensed under the MIT license

"""
Core implementation of aiosqlite proxies
"""

import asyncio
import logging
import sqlite3
import sys
import warnings
from functools import partial
from pathlib import Path
from queue import Empty, Queue
from threading import Thread
from typing import (
    Any,
    AsyncIterator,
    Callable,
    Generator,
    Iterable,
    Optional,
    Type,
    Union,
)
from warnings import warn

from .context import contextmanager
from .cursor import Cursor

__all__ = ["connect", "Connection", "Cursor"]

LOG = logging.getLogger("aiosqlite")

def get_loop(future: asyncio.Future) -> asyncio.AbstractEventLoop:
    if sys.version_info >= (3, 7):
        return future.get_loop()
    else:
        return future._loop

class Connection(Thread):
    def __init__(
        self,
        connector: Callable[[], sqlite3.Connection],
        iter_chunk_size: int,
        loop: Optional[asyncio.AbstractEventLoop] = None,
    ) -> None:
        super().__init__()
        self._running = True
        self._connection: Optional[sqlite3.Connection] = None
        self._connector = connector
        self._tx: Queue = Queue()
        self._iter_chunk_size = iter_chunk_size

        if loop is not None:
            warn(
                "aiosqlite.Connection no longer uses the `loop` parameter",
                DeprecationWarning,
            )

    @property
    def _conn(self) -> sqlite3.Connection:
        if self._connection is None:
            raise ValueError("no active connection")

        return self._connection

    def _execute_insert(
        self, sql: str, parameters: Iterable[Any]
    ) -> Optional[sqlite3.Row]:
        cursor = self._conn.execute(sql, parameters)
        cursor.execute("SELECT last_insert_rowid()")
        return cursor.fetchone()

    def _execute_fetchall(
        self, sql: str, parameters: Iterable[Any]
    ) -> Iterable[sqlite3.Row]:
        cursor = self._conn.execute(sql, parameters)
        return cursor.fetchall()

    def run(self) -> None:
        """
        Execute function calls on a separate thread.

        :meta private:
        """
        while self._running:
            try:
                future, function = self._tx.get(timeout=0.1)
            except Empty:
                continue

            try:
                LOG.debug("executing %s", function)
                result = function()
                LOG.debug("returning %s", result)

                def set_result(fut, result):
                    if not fut.done():
                        fut.set_result(result)

                get_loop(future).call_soon_threadsafe(set_result, future, result)
            except BaseException as e:
                LOG.debug("returning exception %s", e)

                def set_exception(fut, e):
                    if not fut.done():
                        fut.set_exception(e)

                get_loop(future).call_soon_threadsafe(set_exception, future, e)

    async def _execute(self, fn, *args, **kwargs):
        """Queue a function with the given arguments for execution."""
        function = partial(fn, *args, **kwargs)
        future = asyncio.get_event_loop().create_future()

        self._tx.put_nowait((future, function))

        return await future

    async def _connect(self) -> "Connection":
        """Connect to the actual sqlite database."""
        if self._connection is None:
            try:
                self._connection = await self._execute(self._connector)
            except Exception:
                self._running = False
                self._connection = None
                raise

        return self

    def __await__(self) -> Generator[Any, None, "Connection"]:
        self.start()
        return self._connect().__await__()

    async def __aenter__(self) -> "Connection":
        return await self

    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
        await self.close()

    @contextmanager
    async def cursor(self) -> Cursor:
        """Create an aiosqlite cursor wrapping a sqlite3 cursor object."""
        return Cursor(self, await self._execute(self._conn.cursor))

    async def commit(self) -> None:
        """Commit the current transaction."""
        await self._execute(self._conn.commit)

    async def rollback(self) -> None:
        """Roll back the current transaction."""
        await self._execute(self._conn.rollback)

    async def close(self) -> None:
        """Complete queued queries/cursors and close the connection."""
        try:
            await self._execute(self._conn.close)
        except Exception:
            LOG.info("exception occurred while closing connection")
            raise
        finally:
            self._running = False
            self._connection = None

    @contextmanager
    async def execute(self, sql: str, parameters: Iterable[Any] = None) -> Cursor:
        """Helper to create a cursor and execute the given query."""
        if parameters is None:
            parameters = []
        cursor = await self._execute(self._conn.execute, sql, parameters)
        return Cursor(self, cursor)

    @contextmanager
    async def execute_insert(
        self, sql: str, parameters: Iterable[Any] = None
    ) -> Optional[sqlite3.Row]:
        """Helper to insert and get the last_insert_rowid."""
        if parameters is None:
            parameters = []
        return await self._execute(self._execute_insert, sql, parameters)

    @contextmanager
    async def execute_fetchall(
        self, sql: str, parameters: Iterable[Any] = None
    ) -> Iterable[sqlite3.Row]:
        """Helper to execute a query and return all the data."""
        if parameters is None:
            parameters = []
        return await self._execute(self._execute_fetchall, sql, parameters)

    @contextmanager
    async def executemany(
        self, sql: str, parameters: Iterable[Iterable[Any]]
    ) -> Cursor:
        """Helper to create a cursor and execute the given multiquery."""
        cursor = await self._execute(self._conn.executemany, sql, parameters)
        return Cursor(self, cursor)

    @contextmanager
    async def executescript(self, sql_script: str) -> Cursor:
        """Helper to create a cursor and execute a user script."""
        cursor = await self._execute(self._conn.executescript, sql_script)
        return Cursor(self, cursor)

    async def interrupt(self) -> None:
        """Interrupt pending queries."""
        return self._conn.interrupt()

    async def create_function(
        self, name: str, num_params: int, func: Callable, deterministic: bool = False
    ) -> None:
        """
        Create user-defined function that can be later used
        within SQL statements. Must be run within the same thread
        that query executions take place so instead of executing directly
        against the connection, we defer this to `run` function.

        In Python 3.8 and above, if *deterministic* is true, the created
        function is marked as deterministic, which allows SQLite to perform
        additional optimizations. This flag is supported by SQLite 3.8.3 or
        higher, ``NotSupportedError`` will be raised if used with older
        versions.
        """
        if sys.version_info >= (3, 8):
            await self._execute(
                self._conn.create_function,
                name,
                num_params,
                func,
                deterministic=deterministic,
            )
        else:
            if deterministic:
                warnings.warn(
                    "Deterministic function support is only available on "
                    'Python 3.8+. Function "{}" will be registered as '
                    "non-deterministic as per SQLite defaults.".format(name)
                )

            await self._execute(self._conn.create_function, name, num_params, func)

    @property
    def in_transaction(self) -> bool:
        return self._conn.in_transaction

    @property
    def isolation_level(self) -> str:
        return self._conn.isolation_level

    @isolation_level.setter
    def isolation_level(self, value: str) -> None:
        self._conn.isolation_level = value

    @property
    def row_factory(self) -> "Optional[Type]":  # py3.5.2 compat (#24)
        return self._conn.row_factory

    @row_factory.setter
    def row_factory(self, factory: "Optional[Type]") -> None:  # py3.5.2 compat (#24)
        self._conn.row_factory = factory

    @property
    def text_factory(self) -> Type:
        return self._conn.text_factory

    @text_factory.setter
    def text_factory(self, factory: Type) -> None:
        self._conn.text_factory = factory

    @property
    def total_changes(self) -> int:
        return self._conn.total_changes

    async def enable_load_extension(self, value: bool) -> None:
        await self._execute(self._conn.enable_load_extension, value)  # type: ignore

    async def load_extension(self, path: str):
        await self._execute(self._conn.load_extension, path)  # type: ignore

    async def set_progress_handler(
        self, handler: Callable[[], Optional[int]], n: int
    ) -> None:
        await self._execute(self._conn.set_progress_handler, handler, n)

    async def set_trace_callback(self, handler: Callable) -> None:
        await self._execute(self._conn.set_trace_callback, handler)

    async def iterdump(self) -> AsyncIterator[str]:
        """
        Return an async iterator to dump the database in SQL text format.

        Example::

            async for line in db.iterdump():
                ...

        """
        dump_queue: Queue = Queue()

        def dumper():
            try:
                for line in self._conn.iterdump():
                    dump_queue.put_nowait(line)
                dump_queue.put_nowait(None)

            except Exception:
                LOG.exception("exception while dumping db")
                dump_queue.put_nowait(None)
                raise

        fut = self._execute(dumper)
        task = asyncio.ensure_future(fut)

        while True:
            try:
                line: Optional[str] = dump_queue.get_nowait()
                if line is None:
                    break
                yield line

            except Empty:
                if task.done():
                    LOG.warning("iterdump completed unexpectedly")
                    break

                await asyncio.sleep(0.01)

        await task

    async def backup(
        self,
        target: Union["Connection", sqlite3.Connection],
        *,
        pages: int = 0,
        progress: Optional[Callable[[int, int, int], None]] = None,
        name: str = "main",
        sleep: float = 0.250
    ) -> None:
        """
        Make a backup of the current database to the target database.

        Takes either a standard sqlite3 or aiosqlite Connection object as the target.
        """
        if sys.version_info < (3, 7):
            raise RuntimeError("backup() method is only available on Python 3.7+")

        if isinstance(target, Connection):
            target = target._conn

        await self._execute(
            self._conn.backup,
            target,
            pages=pages,
            progress=progress,
            name=name,
            sleep=sleep,
        )

def connect(
    database: Union[str, Path],
    *,
    iter_chunk_size=64,
    loop: Optional[asyncio.AbstractEventLoop] = None,
    **kwargs: Any
) -> Connection:
    """Create and return a connection proxy to the sqlite database."""

    if loop is not None:
        warn(
            "aiosqlite.connect() no longer uses the `loop` parameter",
            DeprecationWarning,
        )

    def connector() -> sqlite3.Connection:
        if isinstance(database, str):
            loc = database
        elif isinstance(database, bytes):
            loc = database.decode("utf-8")
        else:
            loc = str(database)

        return sqlite3.connect(loc, **kwargs)

    return Connection(connector, iter_chunk_size)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Ted Strzalkowski (tedski@gmail.com)`

    tests.unit.modules.mdadm_test
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''

# Import Python libs
from __future__ import absolute_import

# Import Salt Testing libs
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.unit import skipIf, TestCase
from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch

# Import salt libs
import salt.modules.mdadm as mdadm

# Import 3rd-party libs
import salt.ext.six as six

@skipIf(NO_MOCK, NO_MOCK_REASON)
class MdadmTestCase(TestCase, LoaderModuleMockMixin):

    def setup_loader_modules(self):
        return {mdadm: {}}

    def test_create(self):
        mock = MagicMock(return_value='salt')
        with patch.dict(mdadm.__salt__, {'cmd.run': mock}), \
                patch('salt.utils.which', lambda exe: exe):
            ret = mdadm.create(
                    '/dev/md0', 5,
                    devices=['/dev/sdb1', '/dev/sdc1', '/dev/sdd1'],
                    test_mode=False,
                    force=True,
                    chunk=256
            )
            self.assertEqual('salt', ret)
            mock.assert_called_once()
            args, kwargs = mock.call_args
            # expected cmd is
            # mdadm -C /dev/md0 -R -v --chunk 256 --force -l 5 -e default -n 3 /dev/sdb1 /dev/sdc1 /dev/sdd1
            # where args between -v and -l could be in any order
            self.assertEqual(len(args), 1)
            self.assertEqual(len(args[0]), 17)
            self.assertEqual(args[0][:5], [
                'mdadm',
                '-C', '/dev/md0',
                '-R',
                '-v'])
            self.assertEqual(args[0][8:], [
                 '-l', '5',
                 '-e', 'default',
                 '-n', '3',
                 '/dev/sdb1', '/dev/sdc1', '/dev/sdd1'])
            self.assertEqual(sorted(args[0][5:8]), sorted(['--chunk', '256', '--force']))
            self.assertIn('--chunk 256', ' '.join(args[0][5:8]))
            self.assertEqual(kwargs, {'python_shell': False})

    def test_create_test_mode(self):
        mock = MagicMock()
        with patch.dict(mdadm.__salt__, {'cmd.run': mock}):
            ret = mdadm.create(
                    '/dev/md0', 5,
                    devices=['/dev/sdb1', '/dev/sdc1', '/dev/sdd1'],
                    force=True,
                    chunk=256,
                    test_mode=True
            )
            self.assertEqual(sorted('mdadm -C /dev/md0 -R -v --chunk 256 '
                              '--force -l 5 -e default -n 3 '
                              '/dev/sdb1 /dev/sdc1 /dev/sdd1'.split()), sorted(ret.split()))
            assert not mock.called, 'test mode failed, cmd.run called'
<commit_msg>Removed unneeded import.<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Ted Strzalkowski (tedski@gmail.com)`

    tests.unit.modules.mdadm_test
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''

# Import Python libs
from __future__ import absolute_import

# Import Salt Testing libs
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.unit import skipIf, TestCase
from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch

# Import salt libs
import salt.modules.mdadm as mdadm

@skipIf(NO_MOCK, NO_MOCK_REASON)
class MdadmTestCase(TestCase, LoaderModuleMockMixin):

    def setup_loader_modules(self):
        return {mdadm: {}}

    def test_create(self):
        mock = MagicMock(return_value='salt')
        with patch.dict(mdadm.__salt__, {'cmd.run': mock}), \
                patch('salt.utils.which', lambda exe: exe):
            ret = mdadm.create(
                    '/dev/md0', 5,
                    devices=['/dev/sdb1', '/dev/sdc1', '/dev/sdd1'],
                    test_mode=False,
                    force=True,
                    chunk=256
            )
            self.assertEqual('salt', ret)
            mock.assert_called_once()
            args, kwargs = mock.call_args
            # expected cmd is
            # mdadm -C /dev/md0 -R -v --chunk 256 --force -l 5 -e default -n 3 /dev/sdb1 /dev/sdc1 /dev/sdd1
            # where args between -v and -l could be in any order
            self.assertEqual(len(args), 1)
            self.assertEqual(len(args[0]), 17)
            self.assertEqual(args[0][:5], [
                'mdadm',
                '-C', '/dev/md0',
                '-R',
                '-v'])
            self.assertEqual(args[0][8:], [
                 '-l', '5',
                 '-e', 'default',
                 '-n', '3',
                 '/dev/sdb1', '/dev/sdc1', '/dev/sdd1'])
            self.assertEqual(sorted(args[0][5:8]), sorted(['--chunk', '256', '--force']))
            self.assertIn('--chunk 256', ' '.join(args[0][5:8]))
            self.assertEqual(kwargs, {'python_shell': False})

    def test_create_test_mode(self):
        mock = MagicMock()
        with patch.dict(mdadm.__salt__, {'cmd.run': mock}):
            ret = mdadm.create(
                    '/dev/md0', 5,
                    devices=['/dev/sdb1', '/dev/sdc1', '/dev/sdd1'],
                    force=True,
                    chunk=256,
                    test_mode=True
            )
            self.assertEqual(sorted('mdadm -C /dev/md0 -R -v --chunk 256 '
                              '--force -l 5 -e default -n 3 '
                              '/dev/sdb1 /dev/sdc1 /dev/sdd1'.split()), sorted(ret.split()))
            assert not mock.called, 'test mode failed, cmd.run called'
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Based on test code moved here from bottom of tulip/jtlvint.py

SCL; 31 Dec 2011.
"""

import os

from tulip.jtlvint import *
from tulip.polytope import Region
from tulip.prop2part import PropPreservingPartition

class jtlvint_test:

    def setUp(self):
        self.tmpdir = "tmpspec"
        testfile = os.path.join(self.tmpdir, "testjtlvint")
        self.smvfile = testfile + '.smv'
        self.spcfile = testfile + '.spc'
        self.autfile = testfile + '.aut'
    #    env_vars = {'park' : 'boolean', 'cellID' : '{0,...,3,4,5}'}
        env_vars = {'park' : 'boolean', 'cellID' : [0,1,2,3,4,5]}
        sys_disc_vars = {'gear' : '{-1...1}'}
        cont_props = ['X0', 'X1', 'X2', 'X3', 'X4']
        disc_dynamics = PropPreservingPartition()
        region0 = Region('p0', [1, 0, 0, 0, 0])
        region1 = Region('p1', [0, 1, 0, 0, 0])
        region2 = Region('p2', [1, 0, 1, 0, 0])
        region3 = Region('p3', [0, 0, 0, 1, 0])
        region4 = Region('p4', [0, 0, 0, 0, 1])
        region5 = Region('p5', [1, 0, 0, 1, 1])
        disc_dynamics.list_region = [region0, region1, region2,
                                     region3, region4, region5]
        disc_dynamics.num_regions = len(disc_dynamics.list_region)
        disc_dynamics.trans = [[1, 1, 0, 1, 0, 0],
                               [1, 1, 1, 0, 1, 0],
                               [0, 1, 1, 0, 0, 1],
                               [1, 0, 0, 1, 1, 0],
                               [0, 1, 0, 1, 1, 1],
                               [0, 0, 1, 0, 1, 1]]
        disc_dynamics.list_prop_symbol = cont_props
        disc_dynamics.num_prop = len(disc_dynamics.list_prop_symbol)
        disc_props = {'Park' : 'park',
                      'X0d' : 'cellID=0',
                      'X1d' : 'cellID=1',
                      'X2d' : 'cellID=2',
                      'X3d' : 'cellID=3',
                      'X4d' : 'cellID=4',
                      'X5d' : 'gear = 1'}

        assumption = '[]<>(!park) & []<>(!X4d)'
        guarantee = '[]<>(X4d -> X4) & []<>X1 & [](Park -> X0)'
        spec = [assumption, guarantee]

        disc_dynamics=PropPreservingPartition()
        spec[1] = '[]<>(X0d -> X5d)'
        newvarname = generateJTLVInput(env_vars=env_vars,
                                       sys_disc_vars=sys_disc_vars,
                                       spec=spec, disc_props=disc_props,
                                       disc_dynamics=disc_dynamics,
                                       smv_file=self.smvfile,
                                       spc_file=self.spcfile,
                                       file_exist_option='r', verbose=2)

    def tearDown(self):
        os.remove(self.smvfile)
        os.remove(self.spcfile)
        os.remove(self.autfile)
        os.rmdir(self.tmpdir)

    def test_checkRealizability(self):
        assert checkRealizability(smv_file=self.smvfile,
                                  spc_file=self.spcfile,
                                  aut_file=self.autfile,
                                  heap_size='-Xmx128m',
                                  pick_sys_init=False,
                                  file_exist_option='r', verbose=3)

    def test_computeStrategy(self):
        assert computeStrategy(smv_file=self.smvfile,
                               spc_file=self.spcfile,
                               aut_file=self.autfile,
                               heap_size='-Xmx128m',
                               priority_kind='ZYX',
                               init_option=1,
                               file_exist_option='r', verbose=3)

        # assert computeStrategy(smv_file=self.smvfile,
        #                        spc_file=self.spcfile,
        #                        aut_file=self.autfile,
        #                        heap_size='-Xmx128m',
        #                        priority_kind='ZYX',
        #                        init_option=2,
        #                        file_exist_option='r', verbose=3)

        # assert not computeStrategy(smv_file=self.smvfile,
        #                            spc_file=self.spcfile,
        #                            aut_file=self.autfile,
        #                            heap_size='-Xmx128m',
        #                            priority_kind='ZYX',
        #                            init_option=0,
        #                            file_exist_option='r', verbose=3)

    # def test_synthesize(self):
    #     assert synthesize(env_vars=env_vars,
    #                       sys_disc_vars=sys_disc_vars,
    #                       spec=spec, disc_props=disc_props,
    #                       disc_dynamics=disc_dynamics,
    #                       smv_file=self.smvfile,
    #                       spc_file=self.spcfile,
    #                       aut_file=self.autfile,
    #                       heap_size='-Xmx128m', priority_kind=3,
    #                       init_option=init_option,
    #                       file_exist_option='r', verbose=3)

    # def test_getCounterExamples(self):
    #     counter_examples = getCounterExamples(aut_file=self.autfile, verbose=1)
<commit_msg>updated jtlvint tests, removed old tests<commit_after>#!/usr/bin/env python
"""
Based on test code moved here from bottom of tulip/jtlvint.py

SCL; 31 Dec 2011.
Vasu Raman; 4 Sept 2013
"""

import os

from tulip.spec import GRSpec
from tulip.jtlvint import *

class jtlvint_test:

    def test_dumpjtlv(self):
        spec = GRSpec(env_vars="x", sys_vars="y",
                      env_init="x", env_prog="x",
                      sys_init="y", sys_safety=["y -> !y'", "!y -> y'"],
                      sys_prog="y & x")
        specLTL = spec.to_jtlv()
        assumption = specLTL[0]
        guarantee = specLTL[1]
        assert check_gr1(assumption, guarantee, spec.env_vars.keys(), spec.sys_vars.keys())

    def test_check_realizable(self):
        spec = GRSpec(env_vars="x", sys_vars="y",
                      env_init="x", env_prog="x",
                      sys_init="y", sys_safety=["y -> !y'", "!y -> y'"],
                      sys_prog="y & x")
        assert not check_realizable(spec)
        spec.sys_safety = []
        assert check_realizable(spec)

    def test_synthesize(self):
        spec = GRSpec(env_vars="x", sys_vars="y",
                      env_init="x", env_prog="x",
                      sys_init="y",
                      sys_prog=["y & x", "!y"])
        aut = synthesize(spec)
        assert aut is not None<|endoftext|>


<commit_before>import numpy as np
import math

from selfdrive.swaglog import cloudlog
from common.realtime import sec_since_boot
from selfdrive.controls.lib.longitudinal_mpc_lib import libmpc_py
from selfdrive.controls.lib.drive_helpers import LON_MPC_N
from selfdrive.modeld.constants import T_IDXS

class LongitudinalMpc():
  def __init__(self):
    self.reset_mpc()
    self.last_cloudlog_t = 0.0
    self.ts = list(range(10))
    self.status = True
    self.min_a = -1.2
    self.max_a = 1.2

  def reset_mpc(self):
    self.libmpc = libmpc_py.libmpc
    self.libmpc.init(0.0, 1.0, 0.0, 50.0, 10000.0)

    self.mpc_solution = libmpc_py.ffi.new("log_t *")
    self.cur_state = libmpc_py.ffi.new("state_t *")

    self.cur_state[0].x_ego = 0
    self.cur_state[0].v_ego = 0
    self.cur_state[0].a_ego = 0

    self.v_solution = [0.0 for i in range(len(T_IDXS))]
    self.a_solution = [0.0 for i in range(len(T_IDXS))]
    self.j_solution = [0.0 for i in range(len(T_IDXS)-1)]

  def set_accel_limits(self, min_a, max_a):
    self.min_a = min_a
    self.max_a = max_a

  def set_cur_state(self, v, a):
    v_safe = max(v, 1e-2)
    a_safe = min(a, self.max_a - 1e-2)
    a_safe = max(a_safe, self.min_a + 1e-2)
    self.cur_state[0].x_ego = 0.0
    self.cur_state[0].v_ego = v_safe
    self.cur_state[0].a_ego = a_safe

  def update(self, carstate, model, v_cruise):
    v_cruise_clipped = np.clip(v_cruise, self.cur_state[0].v_ego - 10., self.cur_state[0].v_ego + 10.0)
    poss = v_cruise_clipped * np.array(T_IDXS[:LON_MPC_N+1])
    speeds = v_cruise_clipped * np.ones(LON_MPC_N+1)
    accels = np.zeros(LON_MPC_N+1)

    # Calculate mpc
    self.libmpc.run_mpc(self.cur_state, self.mpc_solution,
                        list(poss), list(speeds), list(accels),
                        self.min_a, self.max_a)

    self.v_solution = list(self.mpc_solution.v_ego)
    self.a_solution = list(self.mpc_solution.a_ego)
    self.j_solution = list(self.mpc_solution.j_ego)

    # Reset if NaN or goes through lead car
    nans = any(math.isnan(x) for x in self.mpc_solution[0].v_ego)

    t = sec_since_boot()
    if nans:
      if t > self.last_cloudlog_t + 5.0:
        self.last_cloudlog_t = t
        cloudlog.warning("Longitudinal model mpc reset - nans")
      self.reset_mpc()
<commit_msg>Update long_mpc api for use with sim (#21890)<commit_after>import numpy as np
import math

from selfdrive.swaglog import cloudlog
from common.realtime import sec_since_boot
from selfdrive.controls.lib.longitudinal_mpc_lib import libmpc_py
from selfdrive.controls.lib.drive_helpers import LON_MPC_N
from selfdrive.modeld.constants import T_IDXS

class LongitudinalMpc():
  def __init__(self):
    self.reset_mpc()
    self.last_cloudlog_t = 0.0
    self.ts = list(range(10))
    self.status = True
    self.min_a = -1.2
    self.max_a = 1.2

  def reset_mpc(self):
    self.libmpc = libmpc_py.libmpc
    self.libmpc.init(0.0, 1.0, 0.0, 50.0, 10000.0)

    self.mpc_solution = libmpc_py.ffi.new("log_t *")
    self.cur_state = libmpc_py.ffi.new("state_t *")

    self.cur_state[0].x_ego = 0
    self.cur_state[0].v_ego = 0
    self.cur_state[0].a_ego = 0

    self.v_solution = [0.0 for i in range(len(T_IDXS))]
    self.a_solution = [0.0 for i in range(len(T_IDXS))]
    self.j_solution = [0.0 for i in range(len(T_IDXS)-1)]

  def set_accel_limits(self, min_a, max_a):
    self.min_a = min_a
    self.max_a = max_a

  def set_cur_state(self, v, a):
    v_safe = max(v, 1e-2)
    a_safe = min(a, self.max_a - 1e-2)
    a_safe = max(a_safe, self.min_a + 1e-2)
    self.cur_state[0].x_ego = 0.0
    self.cur_state[0].v_ego = v_safe
    self.cur_state[0].a_ego = a_safe

  def update(self, carstate, radarstate, v_cruise):
    v_cruise_clipped = np.clip(v_cruise, self.cur_state[0].v_ego - 10., self.cur_state[0].v_ego + 10.0)
    poss = v_cruise_clipped * np.array(T_IDXS[:LON_MPC_N+1])
    speeds = v_cruise_clipped * np.ones(LON_MPC_N+1)
    accels = np.zeros(LON_MPC_N+1)
    self.update_with_xva(poss, speeds, accels)

  def update_with_xva(self, poss, speeds, accels):
    # Calculate mpc
    self.libmpc.run_mpc(self.cur_state, self.mpc_solution,
                        list(poss), list(speeds), list(accels),
                        self.min_a, self.max_a)

    self.v_solution = list(self.mpc_solution.v_ego)
    self.a_solution = list(self.mpc_solution.a_ego)
    self.j_solution = list(self.mpc_solution.j_ego)

    # Reset if NaN or goes through lead car
    nans = any(math.isnan(x) for x in self.mpc_solution[0].v_ego)

    t = sec_since_boot()
    if nans:
      if t > self.last_cloudlog_t + 5.0:
        self.last_cloudlog_t = t
        cloudlog.warning("Longitudinal model mpc reset - nans")
      self.reset_mpc()
<|endoftext|>


<commit_before>import asyncio

class BaseApplication:
    def __init__(self, *, config, **kwargs):
        self.config = kwargs.pop('config', None)

    def run_forever(self, **kwargs):
        loop = self.loop

        print("======== Running aioworkers ========\n"
              "(Press CTRL+C to quit)")

        try:
            loop.run_forever()
        except KeyboardInterrupt:
            pass
        loop.close()

class Application(BaseApplication, dict):
    def __init__(self, *, loop, config, **kwargs):
        if loop is None:
            loop = asyncio.get_event_loop()
        self.loop = loop
        super().__init__(config=config)
<commit_msg>factory for app<commit_after>import asyncio

class BaseApplication:
    def __init__(self, *, config, **kwargs):
        self.config = config

    @classmethod
    async def prepare(cls, kwargs):
        return kwargs

    async def init(self):
        pass

    @classmethod
    async def factory(cls, **kwargs):
        kwargs = await cls.prepare(kwargs)
        app = cls(**kwargs)
        await app.init()
        return app

    def run_forever(self, **kwargs):
        loop = self.loop

        print("======== Running aioworkers ========\n"
              "(Press CTRL+C to quit)")

        try:
            loop.run_forever()
        except KeyboardInterrupt:
            pass
        loop.close()

class Application(BaseApplication, dict):
    def __init__(self, *, loop, config, **kwargs):
        if loop is None:
            loop = asyncio.get_event_loop()
        self.loop = loop
        super().__init__(config=config)
<|endoftext|>


<commit_before>import ast
import glob

from underscore import _
from underscore import environment
from underscore import utils
from underscore import variable_visitor
from nose import tools as nt

class KeywordVisitor( variable_visitor._VariableFinder):
    def declare(self, name, _global=False):
        if name != '_' * len(name):
            nt.assert_not_in(name, self._current_frame)
            self._current_frame.add(name, _global)

def testGenerator():
    for filename in glob.glob('examples/*.py'):
        yield _testFile, filename

def _testFile(filename):
    underscored = _(filename, write=False)
    tree = ast.parse(underscored)
    env = environment.Environment(tree)
    visitor = KeywordVisitor(env)
    visitor.visit(tree)
<commit_msg>++ 2.6 compat<commit_after>import ast
import glob

from underscore import _
from underscore import environment
from underscore import utils
from underscore import variable_visitor
from nose import tools as nt

class KeywordVisitor( variable_visitor._VariableFinder):
    def declare(self, name, _global=False):
        if name != '_' * len(name):
            nt.assert_not_in(name, self._current_frame)
            self._current_frame.add(name, _global)

def testGenerator():
    for filename in glob.glob('examples/*.py'):
        # Python 2.6 Compat
        if ('2.7' not in filename or
            ('2.7' in filename and sys.version_info >= (2,7))):
            yield _testFile, filename

def _testFile(filename):
    underscored = _(filename, write=False)
    tree = ast.parse(underscored)
    env = environment.Environment(tree)
    visitor = KeywordVisitor(env)
    visitor.visit(tree)
<|endoftext|>


<commit_before><commit_msg>Sampling freq. is now truly optional.<commit_after><|endoftext|>


<commit_before>
from .main import pony_contexts
from .utils import with_cli_args, PY2

# register default layers

from functools import partial
import click

if not PY2:
    from contextlib import contextmanager
else:
    from contextlib2 import contextmanager

@contextmanager
def ipdb_context(cls):
    import ipdb
    raised = []
    with ipdb.launch_ipdb_on_exception():
        try:
            yield
        except Exception as exc:
            raised.append(exc)
            raise
    raise raised[0]

@with_cli_args
@click.option('--ipdb', 'debug', is_flag=True)
def use_ipdb(debug):
    if debug:
        yield ipdb_context

def use_ipdb_at_test_scope():
    for mgr in use_ipdb():
        mgr = partial(mgr)
        mgr.test_scoped = True
        yield mgr

pony_contexts.extend([
    use_ipdb,
    use_ipdb_at_test_scope,
])
<commit_msg>fix: ipdb_context<commit_after>
from .main import pony_contexts
from .utils import with_cli_args, PY2

# register default layers

from functools import partial
import click

if not PY2:
    from contextlib import contextmanager
else:
    from contextlib2 import contextmanager

@contextmanager
def ipdb_context(cls):
    import ipdb
    raised = []
    with ipdb.launch_ipdb_on_exception():
        try:
            yield
        except Exception as exc:
            raised.append(exc)
            raise
    if raised:
        raise raised[0]

@with_cli_args
@click.option('--ipdb', 'debug', is_flag=True)
def use_ipdb(debug):
    if debug:
        yield ipdb_context

def use_ipdb_at_test_scope():
    for mgr in use_ipdb():
        mgr = partial(mgr)
        mgr.test_scoped = True
        yield mgr

pony_contexts.extend([
    use_ipdb,
    use_ipdb_at_test_scope,
])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Determines the Levenshtein distance between two source files.
"""

from typing import Iterable, Optional, Sequence, cast

from Levenshtein import distance, editops  # type: ignore
from edit_distance import SequenceMatcher  # type: ignore

from sensibility.language import language
from sensibility.vocabulary import Vind
from sensibility import Edit
from sensibility.abram import at_least

# Supplementary Private Use Area B
PUA_B_START = 0x100000

def encode(entries: Iterable[str]) -> str:
    """
    Converts a sequence of tokens into an encoded string, that preserves
    distinctness between tokens. This string can be operated by the
    Levenshtein module.
    """
    # Injective maps of each vocabulary entry to a code point in the private
    # use area (PUA).
    # distance() works on code points, so this effectively makes distance()
    # work on token classes.
    to_index = language.vocabulary.to_index_or_unk
    return ''.join(chr(PUA_B_START + to_index(entry)) for entry in entries)

def tokenwise_distance(file_a: bytes, file_b: bytes, abstract_open_classes=True) -> int:
    """
    Calculates the token-wise Levenshtein distance between two source files.
    """
    if abstract_open_classes:
        seq_a = to_abstraced_tokens(file_a)
        seq_b = to_abstraced_tokens(file_b)
    else:
        seq_a = to_value_stream(file_a)
        seq_b = to_value_stream(file_b)
    return SequenceMatcher(a=seq_a, b=seq_b).distance()

def to_abstraced_tokens(source: bytes) -> Sequence[str]:
    """
    Turns the source code to a sequence of vocabulary tokens.
    """
    return tuple(language.vocabularize(source))

def to_value_stream(source: bytes) -> Sequence[str]:
    """
    Turns the source code to sequence of tokens values.
    """
    return tuple(token.value for token in language.tokenize(source))

class FixEvent:
    def __init__(self, fix: Edit, line_no: int) -> None:
        # Feature request: use the current language for each fix, then to
        # decode the token at instantiation time.
        self.fix = fix
        self.line_no = line_no

    @property
    def mistake(self) -> Edit:
        return -self.fix

    @property
    def old_token(self) -> Optional[str]:
        _code, _pos, _new, old = self.fix.serialize()
        # assert new == f_old and old == f_new
        return language.vocabulary.to_text(old) if old is not None else None

    @property
    def new_token(self) -> Optional[str]:
        # assert new == f_old and old == f_new
        _code, _pos, new, _old = self.fix.serialize()
        return language.vocabulary.to_text(new) if new is not None else None

def determine_edit(file_a: bytes, file_b: bytes) -> Edit:
    return determine_fix_event(file_a, file_b).fix

def determine_fix_event(file_a: bytes, file_b: bytes) -> FixEvent:
    """
    For two source files with Levenshtein distance of one, this returns the
    edit that converts the first file into the second file.
    """
    src_locs, src = lists(language.vocabularize_with_locations(file_a))
    dest = list(language.vocabularize(file_b))
    ops = editops(encode(src), encode(dest))
    # This only works for files with one edit!
    assert len(ops) == 1

    new: Optional[Vind]
    old: Optional[Vind]
    to_index = language.vocabulary.to_index_or_unk

    # Decode editop's format into our "Database-friendly" format.
    (type_name, src_pos, dest_pos), = ops
    if type_name == 'insert':
        code, new, old = 'i', to_index(dest[dest_pos]), None
    elif type_name == 'delete':
        code, new, old = 'x', None, to_index(src[src_pos])
    elif type_name == 'replace':
        code, new, old = 's', to_index(dest[dest_pos]), to_index(src[src_pos])
    else:
        raise ValueError(f'Cannot handle operation: {ops}')

    # Note: in the case of insertions, src_pos will be the index to insert
    # BEFORE, which is exactly what Edit.deserialize wants; src_position also
    # works for deletions, and substitutions.
    edit = Edit.deserialize(code, src_pos, new, old)
    error_token = src_locs[at_least(0, src_pos - 1) if code == 'i' else src_pos]
    return FixEvent(edit, error_token.line)

def lists(it):
    a_list = []
    b_list = []
    for a, b in it:
        a_list.append(a)
        b_list.append(b)
    return a_list, b_list
<commit_msg>Use alternate method to calculate distance.<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Determines the Levenshtein distance between two source files.
"""

from typing import Iterable, Optional, Sequence, Tuple, cast

from Levenshtein import distance, editops  # type: ignore
from edit_distance import SequenceMatcher  # type: ignore

from sensibility.language import language
from sensibility.vocabulary import Vind
from sensibility import Edit
from sensibility.abram import at_least

# Supplementary Private Use Area B
PUA_B_START = 0x100000

EditOp = Tuple[str, int, int, int, int]

def encode(entries: Iterable[str]) -> str:
    """
    Converts a sequence of tokens into an encoded string, that preserves
    distinctness between tokens. This string can be operated by the
    Levenshtein module.
    """
    # Injective maps of each vocabulary entry to a code point in the private
    # use area (PUA).
    # distance() works on code points, so this effectively makes distance()
    # work on token classes.
    to_index = language.vocabulary.to_index_or_unk
    return ''.join(chr(PUA_B_START + to_index(entry)) for entry in entries)

def tokenwise_distance(file_a: bytes, file_b: bytes, abstract_open_classes=True) -> int:
    """
    Calculates the token-wise Levenshtein distance between two source files.
    """
    if abstract_open_classes:
        seq_a = to_abstraced_tokens(file_a)
        seq_b = to_abstraced_tokens(file_b)
    else:
        seq_a = to_value_stream(file_a)
        seq_b = to_value_stream(file_b)
    return SequenceMatcher(a=seq_a, b=seq_b).distance()

class FixEvent:
    def __init__(self, fix: Edit, line_no: int) -> None:
        # Feature request: use the current language for each fix, then to
        # decode the token at instantiation time.
        self.fix = fix
        self.line_no = line_no

    @property
    def mistake(self) -> Edit:
        return -self.fix

    @property
    def old_token(self) -> Optional[str]:
        _code, _pos, _new, old = self.fix.serialize()
        # assert new == f_old and old == f_new
        return language.vocabulary.to_text(old) if old is not None else None

    @property
    def new_token(self) -> Optional[str]:
        # assert new == f_old and old == f_new
        _code, _pos, new, _old = self.fix.serialize()
        return language.vocabulary.to_text(new) if new is not None else None

def determine_edit(file_a: bytes, file_b: bytes, abstract_open_classes=True) -> Edit:
    return determine_fix_event(file_a, file_b, abstract_open_classes).fix

def differences_only(ops) -> Sequence[EditOp]:
    return tuple(op for op in ops if op[0] != 'equal')

def determine_fix_event(file_a: bytes, file_b: bytes, abstract_open_classes=True) -> FixEvent:
    """
    For two source files with Levenshtein distance of one, this returns the
    edit that converts the first file into the second file.
    """
    if abstract_open_classes:
        src_locs, src = lists(language.vocabularize_with_locations(file_a))
        dest = to_abstraced_tokens(file_b)
    else:
        src_locs = []
        src = []
        for token in language.tokenize(file_a):
            src_locs.append(token.location)
            src.append(token.value)
        dest = to_value_stream(file_b)

    ops = differences_only(SequenceMatcher(a=src, b=dest).get_opcodes())
    # This only works for files with one edit!
    assert len(ops) == 1

    new: Optional[Vind]
    old: Optional[Vind]
    to_index = language.vocabulary.to_index_or_unk

    # Decode editop's format into our "Database-friendly" format.
    (type_name, src_pos, src_end, dest_pos, dest_end), = ops
    assert src_end in (src_pos, src_pos + 1)
    assert dest_end in (dest_pos, dest_pos + 1)
    if type_name == 'insert':
        code, new, old = 'i', to_index(dest[dest_pos]), None
        original_token_index = src_pos
        edit_index = dest_pos
    elif type_name == 'delete':
        code, new, old = 'x', None, to_index(src[src_pos])
        original_token_index = edit_index = src_pos
    elif type_name == 'replace':
        code, new, old = 's', to_index(dest[dest_pos]), to_index(src[src_pos])
        original_token_index = edit_index = src_pos
    else:
        raise ValueError(f'Cannot handle operation: {ops}')

    # Note: in the case of insertions, src_pos will be the index to insert
    # BEFORE, which is exactly what Edit.deserialize wants; src_position also
    # works for deletions, and substitutions.
    edit = Edit.deserialize(code, edit_index, new, old)
    error_token = src_locs[original_token_index]

    return FixEvent(edit, error_token.line)

def to_abstraced_tokens(source: bytes) -> Sequence[str]:
    """
    Turns the source code to a sequence of vocabulary tokens.
    """
    return tuple(language.vocabularize(source))

def to_value_stream(source: bytes) -> Sequence[str]:
    """
    Turns the source code to sequence of tokens values.
    """
    return tuple(token.value for token in language.tokenize(source))

def lists(it):
    a_list = []
    b_list = []
    for a, b in it:
        a_list.append(a)
        b_list.append(b)
    return a_list, b_list
<|endoftext|>


<commit_before>import unittest

from algoliasearch.search_client import SearchClient
from algoliasearch.search_index import SearchIndex
from algoliasearch.configs.search_config import SearchConfig
from algoliasearch.http.transporter import Transporter
from algoliasearch.http.requester import Requester

class TestSearchClient(unittest.TestCase):
    def test_init(self):
        config = SearchConfig('foo', 'bar')
        requester = Requester()
        transporter = Transporter(requester, config)
        client = SearchClient(transporter, config)

        self.assertIsInstance(client, SearchClient)

    def test_init_index(self):
        config = SearchConfig('foo', 'bar')
        requester = Requester()
        transporter = Transporter(requester, config)
        client = SearchClient(transporter, config)

        index = client.init_index('foo')

        self.assertIsInstance(index, SearchIndex)

    def test_app_id_getter(self):
        client = SearchClient.create('foo', 'bar')

        self.assertEqual(client.app_id, 'foo')
<commit_msg>Fixes wrong import<commit_after>import unittest

from algoliasearch.search_client import SearchClient
from algoliasearch.search_index import SearchIndex
from algoliasearch.configs import SearchConfig
from algoliasearch.http.transporter import Transporter
from algoliasearch.http.requester import Requester

class TestSearchClient(unittest.TestCase):
    def test_init(self):
        config = SearchConfig('foo', 'bar')
        requester = Requester()
        transporter = Transporter(requester, config)
        client = SearchClient(transporter, config)

        self.assertIsInstance(client, SearchClient)

    def test_init_index(self):
        config = SearchConfig('foo', 'bar')
        requester = Requester()
        transporter = Transporter(requester, config)
        client = SearchClient(transporter, config)

        index = client.init_index('foo')

        self.assertIsInstance(index, SearchIndex)

    def test_app_id_getter(self):
        client = SearchClient.create('foo', 'bar')

        self.assertEqual(client.app_id, 'foo')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import (absolute_import, print_function, unicode_literals)
from acli.output import (output_ascii_table, dash_if_none)

def output_route53_list(output_media=None, zones=None):
    """
    @type output_media: unicode
    @type zones: list | dict
    """
    if isinstance(zones, dict):
        zones = [zones]
    for hosted_zone_dict in zones:
            if output_media == 'console':
                td = list()
                td.append(['id', 'name',
                           'count', 'comment',
                           'private zone'])
                for hosted_zone in hosted_zone_dict.get('HostedZones'):
                    zone_id = dash_if_none(hosted_zone.get('Id'))
                    zone_name = hosted_zone.get('Name')
                    record_count = str(hosted_zone.get('ResourceRecordSetCount'))
                    comment = hosted_zone.get('Config').get('Comment')
                    private_zone = str(hosted_zone.get('Config').get('PrivateZone'))
                    td.append([zone_id,
                               zone_name,
                               record_count,
                               comment,
                               private_zone])
                output_ascii_table(table_title="Route53 Zones",
                                   table_data=td,
                                   inner_heading_row_border=True)
    exit(0)

def get_record_set_values(resource_records):
    """
    @type resource_records: list
    """
    out = list()
    for record in resource_records:
        out.append(record.get('Value'))
    return out

def output_route53_info(output_media=None, zone=None, record_sets=None):
    """
    @type output_media: unicode
    @type zone: zone
    @type record_sets: ResourceRecordSets
    """
    if output_media == 'console':
        td = list()
        td.append(['id', zone['HostedZone']['Id']])
        td.append(['Name', zone['HostedZone']['Name']])
        td.append(['Count', str(zone['HostedZone']['ResourceRecordSetCount'])])
        td.append(['Comment', zone['HostedZone']['Config']['Comment']])
        td.append(['Private', str(zone['HostedZone']['Config']['PrivateZone'])])
        td.append(['Name Servers', "\n".join(zone['DelegationSet']['NameServers'])])
        td.append(['Records', ' '])
        td.append(['{0}'.format("-" * 12), '{0}'.format("-" * 20)])
        for record_set in record_sets['ResourceRecordSets']:
            td.append(['Name', record_set['Name']])
            td.append([' Type', record_set['Type']])
            td.append([' TTL', str(record_set['TTL'])])
            td.append([' Values', "\n".join(get_record_set_values(record_set['ResourceRecords']))])
        output_ascii_table(table_title="Zone Info",
                           table_data=td)
    exit(0)
<commit_msg>sort by name.<commit_after># -*- coding: utf-8 -*-
from __future__ import (absolute_import, print_function, unicode_literals)
from acli.output import (output_ascii_table, dash_if_none)

def output_route53_list(output_media=None, zones=None):
    """
    @type output_media: unicode
    @type zones: list | dict
    """
    if isinstance(zones, dict):
        zones = [zones]
    for hosted_zone_dict in zones:
            if output_media == 'console':
                td = list()
                td.append(['id', 'name',
                           'count', 'comment',
                           'private zone'])
                hosted_zones = sorted(hosted_zone_dict.get('HostedZones'), key=lambda k: k['Name'])
                for hosted_zone in hosted_zones:
                    zone_id = dash_if_none(hosted_zone.get('Id'))
                    zone_name = hosted_zone.get('Name')
                    record_count = str(hosted_zone.get('ResourceRecordSetCount'))
                    comment = hosted_zone.get('Config').get('Comment')
                    private_zone = str(hosted_zone.get('Config').get('PrivateZone'))
                    td.append([zone_id,
                               zone_name,
                               record_count,
                               comment,
                               private_zone])
                output_ascii_table(table_title="Route53 Zones",
                                   table_data=td,
                                   inner_heading_row_border=True)
    exit(0)

def get_record_set_values(resource_records):
    """
    @type resource_records: list
    """
    out = list()
    for record in resource_records:
        out.append(record.get('Value'))
    return out

def output_route53_info(output_media=None, zone=None, record_sets=None):
    """
    @type output_media: unicode
    @type zone: zone
    @type record_sets: ResourceRecordSets
    """
    if output_media == 'console':
        td = list()
        td.append(['id', zone['HostedZone']['Id']])
        td.append(['Name', zone['HostedZone']['Name']])
        td.append(['Count', str(zone['HostedZone']['ResourceRecordSetCount'])])
        td.append(['Comment', zone['HostedZone']['Config']['Comment']])
        td.append(['Private', str(zone['HostedZone']['Config']['PrivateZone'])])
        td.append(['Name Servers', "\n".join(zone['DelegationSet']['NameServers'])])
        td.append(['Records', ' '])
        td.append(['{0}'.format("-" * 12), '{0}'.format("-" * 20)])
        for record_set in record_sets['ResourceRecordSets']:
            td.append(['Name', record_set['Name']])
            td.append([' Type', record_set['Type']])
            td.append([' TTL', str(record_set['TTL'])])
            td.append([' Values', "\n".join(get_record_set_values(record_set['ResourceRecords']))])
        output_ascii_table(table_title="Zone Info",
                           table_data=td)
    exit(0)
<|endoftext|>


<commit_before>"""
Wrapper module for the Pyte terminal emulator
"""
from collections import deque, namedtuple
from itertools import islice
import math

from . import pyte
from .pyte import modes

class PyteTerminalEmulator():
    """
    Adapter for the pyte terminal emulator
    """
    def __init__(self, cols, lines, history, ratio):
        # Double history size due to pyte splitting it between two queues
        # resulting in only having half the scrollback as expected
        self._screen = CustomHistoryScreen(cols, lines, history * 2, ratio)
        self._bytestream = pyte.ByteStream()
        self._bytestream.attach(self._screen)
        self._modified = True

    def feed(self, data):
        self._screen.scroll_to_bottom()
        self._bytestream.feed(data)
        self._modified = True
        print(self._screen.tabstops)

    def resize(self, lines, cols):
        self._screen.scroll_to_bottom()
        dirty_lines = max(lines, self._screen.lines)
        self._screen.dirty.update(range(dirty_lines))
        self._modified = True
        return self._screen.resize(lines, cols)

    def prev_page(self):
        self._screen.prev_page()
        self._screen.ensure_screen_width()
        self._modified = True

    def next_page(self):
        self._screen.next_page()
        self._screen.ensure_screen_width()
        self._modified = True

    def dirty_lines(self):
        dirty_lines = {}
        nb_dirty_lines = len(self._screen.dirty)
        if nb_dirty_lines > 0:
            display = self._screen.display
            for line in self._screen.dirty:
                if line >= len(display):
                    # This happens when screen is resized smaller
                    break
                else:
                    dirty_lines[line] = display[line]

        return dirty_lines

    def clear_dirty(self):
        self._modified = False
        return self._screen.dirty.clear()

    def cursor(self):
        cursor = self._screen.cursor
        if cursor:
            return (cursor.y, cursor.x)

        return (0, 0)

    def color_map(self, lines):
        return convert_pyte_buffer_to_colormap(self._screen.buffer, lines)

    def display(self):
        return self._screen.display

    def modified(self):
        return self._modified

    def bracketed_paste_mode_enabled(self):
        return (2004 << 5) in self._screen.mode

    def application_mode_enabled(self):
        return False

    def nb_lines(self):
        return self._screen.lines

History = namedtuple("History", "top bottom ratio size position")
Margins = namedtuple("Margins", "top bottom")

class CustomHistoryScreen(pyte.DiffScreen):
    """
    Custom history screen customized for this plugin. Basically a copy of the
    standard pyte history screen but with some optimizations.
    """
    def __init__(self, columns, lines, history, ratio):
        self.history = History(deque(maxlen=history // 2),
                               deque(maxlen=history),
                               float(ratio),
                               history,
                               history)

        super(CustomHistoryScreen, self).__init__(columns, lines)

        # We do not agree with pyte about default tabstops
        self.tabstops = set(range(8, self.columns, 8))

    def reset(self):
        super(CustomHistoryScreen, self).reset()

        # Reset tab stops
        self.tabstops = set(range(8, self.columns, 8))

    def scroll_to_bottom(self):
        """
        Ensure a screen is at the bottom of the history buffer
        """
        while self.history.position < self.history.size:
            self.next_page()

    def ensure_screen_width(self):
        """
        Ensure all lines on a screen have proper width columns. Extra characters
        are truncated, missing characters are filled with whitespace.
        """
        for idx, line in enumerate(self.buffer):
            if len(line) > self.columns:
                self.buffer[idx] = line[:self.columns]
            elif len(line) < self.columns:
                self.buffer[idx] = line + take(self.columns - len(line),
                                               self.default_line)

        # If we're at the bottom of the history buffer and `DECTCEM`
        # mode is set -- show the cursor.
        self.cursor.hidden = not (
            abs(self.history.position - self.history.size) < self.lines and
            modes.DECTCEM in self.mode
        )

    def reset_history(self):
        self.history.top.clear()
        self.history.bottom.clear()
        self.history = self.history._replace(position=self.history.size)

    def reset(self):
        """
        Overloaded to reset screen history state: history position
        is reset to bottom of both queues;  queues themselves are
        emptied.
        """
        super(CustomHistoryScreen, self).reset()
        self.reset_history()

    def erase_in_display(self, how=0):
        """
        Overloaded to reset history state
        """
        super(CustomHistoryScreen, self).erase_in_display(how)

        if how == 3:
            self.reset_history()

    def index(self):
        """
        Overloaded to update top history with the removed lines
        """
        top, bottom = self.margins

        if self.cursor.y == bottom:
            self.history.top.append(self.buffer[top])

        super(CustomHistoryScreen, self).index()

    def reverse_index(self):
        """
        Overloaded to update bottom history with the removed lines
        """
        top, bottom = self.margins

        if self.cursor.y == top:
            self.history.bottom.append(self.buffer[bottom])

        super(CustomHistoryScreen, self).reverse_index()

    def prev_page(self):
        """
        Move the screen page up through the history buffer
        """
        if self.history.position > self.lines and self.history.top:
            mid = min(len(self.history.top),
                      int(math.ceil(self.lines * self.history.ratio)))

            self.history.bottom.extendleft(reversed(self.buffer[-mid:]))
            self.history = self.history \
                ._replace(position=self.history.position - self.lines)

            self.buffer[:] = list(reversed([
                self.history.top.pop() for _ in range(mid)
            ])) + self.buffer[:-mid]

            self.dirty = set(range(self.lines))

    def next_page(self):
        """
        Move the screen page down through the history buffer
        """
        if self.history.position < self.history.size and self.history.bottom:
            mid = min(len(self.history.bottom),
                      int(math.ceil(self.lines * self.history.ratio)))

            self.history.top.extend(self.buffer[:mid])
            self.history = self.history \
                ._replace(position=self.history.position + self.lines)

            self.buffer[:] = self.buffer[mid:] + [
                self.history.bottom.popleft() for _ in range(mid)
            ]

            self.dirty = set(range(self.lines))

    def resize(self, lines=None, columns=None):
        lines = lines or self.lines
        columns = columns or self.columns

        # First resize the lines:
        line_diff = self.lines - lines

        # a) if the current display size is less than the requested
        #    size, add lines to the bottom.
        if line_diff < 0:
            self.buffer.extend(take(self.columns, self.default_line)
                               for _ in range(line_diff, 0))
        # b) if the current display size is greater than requested
        #    size, take lines off the top.
        elif line_diff > 0:
            # JW tweak - if we only have spaces in the bottom of the screen
            # remove those lines instead
            disp = self.display[-line_diff:]
            contents = "".join(disp)
            if contents.isspace():
                self.buffer[-line_diff:] = ()
            else:
                self.buffer[:line_diff] = ()

        # Then resize the columns:
        col_diff = self.columns - columns

        # a) if the current display size is less than the requested
        #    size, expand each line to the new size.
        if col_diff < 0:
            for y in range(lines):
                self.buffer[y].extend(take(abs(col_diff), self.default_line))
        # b) if the current display size is greater than requested
        #    size, trim each line from the right to the new size.
        elif col_diff > 0:
            for line in self.buffer:
                del line[columns:]

        self.lines, self.columns = lines, columns
        self.margins = Margins(0, self.lines - 1)

        # JW tweak - move cursor upwards if its out of bounds do not reset it
        self.ensure_bounds(use_margins=True)

        # Update tabstops to new screen size
        self.tabstops = set(range(8, self.columns, 8))

def take(n, iterable):
    """Returns first n items of the iterable as a list."""
    return list(islice(iterable, n))

def convert_go_renditions_to_colormap(renditions, renditions_store, lines):
    out = []
    for line in renditions:
        line_rends = []
        for rendition in line:
            line_rends.append(renditions_store[rendition])
        out.append(line_rends)
    print(out)

def convert_pyte_buffer_to_colormap(buffer, lines):
    """
    Convert a pyte buffer to a simple colors
    """
    color_map = {}
    for line_index in lines:
        # There may be lines outside the buffer after terminal was resized.
        # These are considered blank.
        if line_index > len(buffer) - 1:
            break

        # Get line and process all colors on that. If there are multiple
        # continuous fields with same color we want to combine them for
        # optimization and because it looks better when rendered in ST3.
        line = buffer[line_index]
        line_len = len(line)
        if line_len == 0:
            continue

        # Initialize vars to keep track of continuous colors
        last_bg = line[0].bg
        if last_bg == "default":
            last_bg = "black"

        last_fg = line[0].fg
        if last_fg == "default":
            last_fg = "white"

        if line[0].reverse:
            last_color = (last_fg, last_bg)
        else:
            last_color = (last_bg, last_fg)

        last_index = 0
        field_length = 0

        char_index = 0
        for char in line:
            # Default bg is black
            if char.bg is "default":
                bg = "black"
            else:
                bg = char.bg

            # Default fg is white
            if char.fg is "default":
                fg = "white"
            else:
                fg = char.fg

            if char.reverse:
                color = (fg, bg)
            else:
                color = (bg, fg)

            if last_color == color:
                field_length = field_length + 1
            else:
                color_dict = {"color": last_color, "field_length": field_length}

                if last_color != ("black", "white"):
                    if line_index not in color_map:
                        color_map[line_index] = {}
                    color_map[line_index][last_index] = color_dict

                last_color = color
                last_index = char_index
                field_length = 1

            # Check if last color was active to the end of screen
            if last_color != ("black", "white"):
                color_dict = {"color": last_color, "field_length": field_length}
                if line_index not in color_map:
                    color_map[line_index] = {}
                color_map[line_index][last_index] = color_dict

            char_index = char_index + 1
    return color_map
<commit_msg>Remove debug print from last commit<commit_after>"""
Wrapper module for the Pyte terminal emulator
"""
from collections import deque, namedtuple
from itertools import islice
import math

from . import pyte
from .pyte import modes

class PyteTerminalEmulator():
    """
    Adapter for the pyte terminal emulator
    """
    def __init__(self, cols, lines, history, ratio):
        # Double history size due to pyte splitting it between two queues
        # resulting in only having half the scrollback as expected
        self._screen = CustomHistoryScreen(cols, lines, history * 2, ratio)
        self._bytestream = pyte.ByteStream()
        self._bytestream.attach(self._screen)
        self._modified = True

    def feed(self, data):
        self._screen.scroll_to_bottom()
        self._bytestream.feed(data)
        self._modified = True

    def resize(self, lines, cols):
        self._screen.scroll_to_bottom()
        dirty_lines = max(lines, self._screen.lines)
        self._screen.dirty.update(range(dirty_lines))
        self._modified = True
        return self._screen.resize(lines, cols)

    def prev_page(self):
        self._screen.prev_page()
        self._screen.ensure_screen_width()
        self._modified = True

    def next_page(self):
        self._screen.next_page()
        self._screen.ensure_screen_width()
        self._modified = True

    def dirty_lines(self):
        dirty_lines = {}
        nb_dirty_lines = len(self._screen.dirty)
        if nb_dirty_lines > 0:
            display = self._screen.display
            for line in self._screen.dirty:
                if line >= len(display):
                    # This happens when screen is resized smaller
                    break
                else:
                    dirty_lines[line] = display[line]

        return dirty_lines

    def clear_dirty(self):
        self._modified = False
        return self._screen.dirty.clear()

    def cursor(self):
        cursor = self._screen.cursor
        if cursor:
            return (cursor.y, cursor.x)

        return (0, 0)

    def color_map(self, lines):
        return convert_pyte_buffer_to_colormap(self._screen.buffer, lines)

    def display(self):
        return self._screen.display

    def modified(self):
        return self._modified

    def bracketed_paste_mode_enabled(self):
        return (2004 << 5) in self._screen.mode

    def application_mode_enabled(self):
        return False

    def nb_lines(self):
        return self._screen.lines

History = namedtuple("History", "top bottom ratio size position")
Margins = namedtuple("Margins", "top bottom")

class CustomHistoryScreen(pyte.DiffScreen):
    """
    Custom history screen customized for this plugin. Basically a copy of the
    standard pyte history screen but with some optimizations.
    """
    def __init__(self, columns, lines, history, ratio):
        self.history = History(deque(maxlen=history // 2),
                               deque(maxlen=history),
                               float(ratio),
                               history,
                               history)

        super(CustomHistoryScreen, self).__init__(columns, lines)

        # We do not agree with pyte about default tabstops
        self.tabstops = set(range(8, self.columns, 8))

    def reset(self):
        super(CustomHistoryScreen, self).reset()

        # Reset tab stops
        self.tabstops = set(range(8, self.columns, 8))

    def scroll_to_bottom(self):
        """
        Ensure a screen is at the bottom of the history buffer
        """
        while self.history.position < self.history.size:
            self.next_page()

    def ensure_screen_width(self):
        """
        Ensure all lines on a screen have proper width columns. Extra characters
        are truncated, missing characters are filled with whitespace.
        """
        for idx, line in enumerate(self.buffer):
            if len(line) > self.columns:
                self.buffer[idx] = line[:self.columns]
            elif len(line) < self.columns:
                self.buffer[idx] = line + take(self.columns - len(line),
                                               self.default_line)

        # If we're at the bottom of the history buffer and `DECTCEM`
        # mode is set -- show the cursor.
        self.cursor.hidden = not (
            abs(self.history.position - self.history.size) < self.lines and
            modes.DECTCEM in self.mode
        )

    def reset_history(self):
        self.history.top.clear()
        self.history.bottom.clear()
        self.history = self.history._replace(position=self.history.size)

    def reset(self):
        """
        Overloaded to reset screen history state: history position
        is reset to bottom of both queues;  queues themselves are
        emptied.
        """
        super(CustomHistoryScreen, self).reset()
        self.reset_history()

    def erase_in_display(self, how=0):
        """
        Overloaded to reset history state
        """
        super(CustomHistoryScreen, self).erase_in_display(how)

        if how == 3:
            self.reset_history()

    def index(self):
        """
        Overloaded to update top history with the removed lines
        """
        top, bottom = self.margins

        if self.cursor.y == bottom:
            self.history.top.append(self.buffer[top])

        super(CustomHistoryScreen, self).index()

    def reverse_index(self):
        """
        Overloaded to update bottom history with the removed lines
        """
        top, bottom = self.margins

        if self.cursor.y == top:
            self.history.bottom.append(self.buffer[bottom])

        super(CustomHistoryScreen, self).reverse_index()

    def prev_page(self):
        """
        Move the screen page up through the history buffer
        """
        if self.history.position > self.lines and self.history.top:
            mid = min(len(self.history.top),
                      int(math.ceil(self.lines * self.history.ratio)))

            self.history.bottom.extendleft(reversed(self.buffer[-mid:]))
            self.history = self.history \
                ._replace(position=self.history.position - self.lines)

            self.buffer[:] = list(reversed([
                self.history.top.pop() for _ in range(mid)
            ])) + self.buffer[:-mid]

            self.dirty = set(range(self.lines))

    def next_page(self):
        """
        Move the screen page down through the history buffer
        """
        if self.history.position < self.history.size and self.history.bottom:
            mid = min(len(self.history.bottom),
                      int(math.ceil(self.lines * self.history.ratio)))

            self.history.top.extend(self.buffer[:mid])
            self.history = self.history \
                ._replace(position=self.history.position + self.lines)

            self.buffer[:] = self.buffer[mid:] + [
                self.history.bottom.popleft() for _ in range(mid)
            ]

            self.dirty = set(range(self.lines))

    def resize(self, lines=None, columns=None):
        lines = lines or self.lines
        columns = columns or self.columns

        # First resize the lines:
        line_diff = self.lines - lines

        # a) if the current display size is less than the requested
        #    size, add lines to the bottom.
        if line_diff < 0:
            self.buffer.extend(take(self.columns, self.default_line)
                               for _ in range(line_diff, 0))
        # b) if the current display size is greater than requested
        #    size, take lines off the top.
        elif line_diff > 0:
            # JW tweak - if we only have spaces in the bottom of the screen
            # remove those lines instead
            disp = self.display[-line_diff:]
            contents = "".join(disp)
            if contents.isspace():
                self.buffer[-line_diff:] = ()
            else:
                self.buffer[:line_diff] = ()

        # Then resize the columns:
        col_diff = self.columns - columns

        # a) if the current display size is less than the requested
        #    size, expand each line to the new size.
        if col_diff < 0:
            for y in range(lines):
                self.buffer[y].extend(take(abs(col_diff), self.default_line))
        # b) if the current display size is greater than requested
        #    size, trim each line from the right to the new size.
        elif col_diff > 0:
            for line in self.buffer:
                del line[columns:]

        self.lines, self.columns = lines, columns
        self.margins = Margins(0, self.lines - 1)

        # JW tweak - move cursor upwards if its out of bounds do not reset it
        self.ensure_bounds(use_margins=True)

        # Update tabstops to new screen size
        self.tabstops = set(range(8, self.columns, 8))

def take(n, iterable):
    """Returns first n items of the iterable as a list."""
    return list(islice(iterable, n))

def convert_go_renditions_to_colormap(renditions, renditions_store, lines):
    out = []
    for line in renditions:
        line_rends = []
        for rendition in line:
            line_rends.append(renditions_store[rendition])
        out.append(line_rends)
    print(out)

def convert_pyte_buffer_to_colormap(buffer, lines):
    """
    Convert a pyte buffer to a simple colors
    """
    color_map = {}
    for line_index in lines:
        # There may be lines outside the buffer after terminal was resized.
        # These are considered blank.
        if line_index > len(buffer) - 1:
            break

        # Get line and process all colors on that. If there are multiple
        # continuous fields with same color we want to combine them for
        # optimization and because it looks better when rendered in ST3.
        line = buffer[line_index]
        line_len = len(line)
        if line_len == 0:
            continue

        # Initialize vars to keep track of continuous colors
        last_bg = line[0].bg
        if last_bg == "default":
            last_bg = "black"

        last_fg = line[0].fg
        if last_fg == "default":
            last_fg = "white"

        if line[0].reverse:
            last_color = (last_fg, last_bg)
        else:
            last_color = (last_bg, last_fg)

        last_index = 0
        field_length = 0

        char_index = 0
        for char in line:
            # Default bg is black
            if char.bg is "default":
                bg = "black"
            else:
                bg = char.bg

            # Default fg is white
            if char.fg is "default":
                fg = "white"
            else:
                fg = char.fg

            if char.reverse:
                color = (fg, bg)
            else:
                color = (bg, fg)

            if last_color == color:
                field_length = field_length + 1
            else:
                color_dict = {"color": last_color, "field_length": field_length}

                if last_color != ("black", "white"):
                    if line_index not in color_map:
                        color_map[line_index] = {}
                    color_map[line_index][last_index] = color_dict

                last_color = color
                last_index = char_index
                field_length = 1

            # Check if last color was active to the end of screen
            if last_color != ("black", "white"):
                color_dict = {"color": last_color, "field_length": field_length}
                if line_index not in color_map:
                    color_map[line_index] = {}
                color_map[line_index][last_index] = color_dict

            char_index = char_index + 1
    return color_map
<|endoftext|>


<commit_before>import unittest
import re
from steam import items
from steam import sim

class BaseTestCase(unittest.TestCase):
    TEST_APP = (440, 'en_US')     # TF2 English catalog
    ITEM_IN_CATALOG = 344         # Crocleather Slouch
    ITEM_NOT_IN_CATALOG = 1       # Bottle
    TEST_ID64 = 76561198014028523 # Yours truly

    TEST_APP_NO_TAGS = (570, 'en_US')     # Dota 2 English catalog
    ITEM_IN_NO_TAGS_CATALOG = 4097         # Arctic Hunter's Ice Axe

class AssetTestCase(BaseTestCase):
    def test_asset_contains(self):
        assets = items.assets(*self.TEST_APP)
        self.assertTrue(self.ITEM_IN_CATALOG in assets)
        self.assertFalse(self.ITEM_NOT_IN_CATALOG in assets)
        schema = items.schema(*self.TEST_APP)
        self.assertTrue(schema[self.ITEM_IN_CATALOG] in assets)
        self.assertFalse(schema[self.ITEM_NOT_IN_CATALOG] in assets)

    def test_asset_has_tags(self):
        assets_with_tags = items.assets(*self.TEST_APP)
        self.assertGreater(len(assets_with_tags.tags), 0)

    def test_asset_has_no_tags(self):
        assets_without_tags = items.assets(*self.TEST_APP_NO_TAGS)
        self.assertEqual(len(assets_without_tags.tags), 0)

    def test_asset_item_has_tags(self):
        assets_with_tags = items.assets(*self.TEST_APP)
        asset_item_with_tags = assets_with_tags[self.ITEM_IN_CATALOG]
        self.assertGreater(len(asset_item_with_tags.tags), 0)

    def test_asset_item_has_no_tags(self):
        assets_without_tags = items.assets(*self.TEST_APP_NO_TAGS)
        asset_item_without_tags = assets_without_tags[self.ITEM_IN_NO_TAGS_CATALOG]
        self.assertEqual(len(asset_item_without_tags.tags), 0)

class InventoryBaseTestCase(BaseTestCase):
    @classmethod
    def setUpClass(cls):
        cls._schema = items.schema(*cls.TEST_APP)
        cls._inv = items.inventory(cls.TEST_APP[0], cls.TEST_ID64, cls._schema)

class ItemTestCase(InventoryBaseTestCase):
    def test_position(self):
        for item in self._inv:
            self.assertLessEqual(item.position, self._inv.cells_total)

    def test_equipped(self):
        for item in self._inv:
            self.assertNotIn(0, item.equipped.keys())
            self.assertNotIn(65535, item.equipped.values())

    def test_name(self):
        # Since sim names are generated by Valve we'll test against those for consistency
        sim_inv = sim.inventory(sim.inventory_context(self.TEST_ID64)[440], self.TEST_ID64)
        # steamodd adds craft numbers to all names, valve doesn't, so they should be stripped
        # steamodd doesn't add crate series to names, valve does, so they should be stripped as well
        cn_exp = re.compile(r" (?:Series )?#\d+$")

        sim_names = set()
        for item in sim_inv:
            # Removes quotes in case of custom name (steamodd leaves that aesthetic choice to the user)
            name = item.full_name.strip("'")
            sim_names.add(cn_exp.sub('', name))

        our_names = set([cn_exp.sub('', item.custom_name or item.full_name) for item in self._inv])

        self.assertEqual(our_names, sim_names)

    def test_attributes(self):
        # Similarly to the name, we'll test against Valve's strings to check for consistency in the math.
        sim_inv = sim.inventory(sim.inventory_context(self.TEST_ID64)[440], self.TEST_ID64)

        schema_attr_exps = []
        for attr in self._schema.attributes:
            if not attr.description:
                continue

            desc = attr.description.strip()
            exp = re.escape(desc).replace("\\%s1", r"[\d-]+")

            schema_attr_exps.append(re.compile(exp))

        sim_attrs = {}
        for item in sim_inv:
            sim_attrs.setdefault(item.id, set())

            for attr in item:
                # Due to lack of contextual data, we'll have to do fuzzy matching to separate actual attrs from fluff/descriptions
                desc = attr.description.strip()
                if desc:
                    # Stop processing if we hit item set attrs, for now
                    if desc.startswith("Item Set Bonus:"):
                        break

                    # Valve for some reason insists on this being attached by the client, since they're not actually attached we skip it.
                    if desc == "Given to valuable Community Contributors":
                        continue

                    for exp in schema_attr_exps:
                        if exp.match(desc):
                            sim_attrs[item.id].add(desc)
                            break

        for item in self._inv:
            # Ignore hidden, special (for now) and date values (timestamp formatting is an eternal battle, let it not be fought on these hallowed testgrounds)
            attrs = set([attr.formatted_description for attr in item if not attr.hidden and attr.value_type not in ("date", "particle_index")])
            self.assertTrue(item.id in sim_attrs)
            self.assertEqual(attrs, sim_attrs[item.id])

class InventoryTestCase(InventoryBaseTestCase):
    def test_cell_count(self):
        self.assertLessEqual(len(list(self._inv)), self._inv.cells_total)
<commit_msg>Workaround sim returning incorrect strange item names<commit_after>import unittest
import re
from steam import items
from steam import sim

class BaseTestCase(unittest.TestCase):
    TEST_APP = (440, 'en_US')     # TF2 English catalog
    ITEM_IN_CATALOG = 344         # Crocleather Slouch
    ITEM_NOT_IN_CATALOG = 1       # Bottle
    TEST_ID64 = 76561198014028523 # Yours truly

    TEST_APP_NO_TAGS = (570, 'en_US')     # Dota 2 English catalog
    ITEM_IN_NO_TAGS_CATALOG = 4097         # Arctic Hunter's Ice Axe

class AssetTestCase(BaseTestCase):
    def test_asset_contains(self):
        assets = items.assets(*self.TEST_APP)
        self.assertTrue(self.ITEM_IN_CATALOG in assets)
        self.assertFalse(self.ITEM_NOT_IN_CATALOG in assets)
        schema = items.schema(*self.TEST_APP)
        self.assertTrue(schema[self.ITEM_IN_CATALOG] in assets)
        self.assertFalse(schema[self.ITEM_NOT_IN_CATALOG] in assets)

    def test_asset_has_tags(self):
        assets_with_tags = items.assets(*self.TEST_APP)
        self.assertGreater(len(assets_with_tags.tags), 0)

    def test_asset_has_no_tags(self):
        assets_without_tags = items.assets(*self.TEST_APP_NO_TAGS)
        self.assertEqual(len(assets_without_tags.tags), 0)

    def test_asset_item_has_tags(self):
        assets_with_tags = items.assets(*self.TEST_APP)
        asset_item_with_tags = assets_with_tags[self.ITEM_IN_CATALOG]
        self.assertGreater(len(asset_item_with_tags.tags), 0)

    def test_asset_item_has_no_tags(self):
        assets_without_tags = items.assets(*self.TEST_APP_NO_TAGS)
        asset_item_without_tags = assets_without_tags[self.ITEM_IN_NO_TAGS_CATALOG]
        self.assertEqual(len(asset_item_without_tags.tags), 0)

class InventoryBaseTestCase(BaseTestCase):
    @classmethod
    def setUpClass(cls):
        cls._schema = items.schema(*cls.TEST_APP)
        cls._inv = items.inventory(cls.TEST_APP[0], cls.TEST_ID64, cls._schema)

class ItemTestCase(InventoryBaseTestCase):
    def test_position(self):
        for item in self._inv:
            self.assertLessEqual(item.position, self._inv.cells_total)

    def test_equipped(self):
        for item in self._inv:
            self.assertNotIn(0, item.equipped.keys())
            self.assertNotIn(65535, item.equipped.values())

    def test_name(self):
        # Since sim names are generated by Valve we'll test against those for consistency
        sim_inv = sim.inventory(sim.inventory_context(self.TEST_ID64)[440], self.TEST_ID64)
        # steamodd adds craft numbers to all names, valve doesn't, so they should be stripped
        # steamodd doesn't add crate series to names, valve does, so they should be stripped as well
        cn_exp = re.compile(r" (?:Series )?#\d+$")

        sim_names = set()
        for item in sim_inv:
            # Removes quotes in case of custom name (steamodd leaves that aesthetic choice to the user)
            name = item.full_name.strip("'")

            # Don't even bother with strange items right now. I'm tired of unit tests failing whenever
            # the inventory does and no one from valve responds when I tell them of the issue. If it does
            # get fixed feel free to remove this as it is definitely a WORKAROUND.
            # Removes quotes in case of custom name (steamodd leaves that aesthetic choice to the user)
            if not name.startswith("Strange "):
                sim_names.add(cn_exp.sub('', name))

        # See the above WORKAROUND about strange items and remove if/when it's fixed.
        our_names = set([cn_exp.sub('', item.custom_name or item.full_name) for item in self._inv if item.quality[1] != "strange" or item.custom_name])

        self.assertEqual(our_names, sim_names)

    def test_attributes(self):
        # Similarly to the name, we'll test against Valve's strings to check for consistency in the math.
        sim_inv = sim.inventory(sim.inventory_context(self.TEST_ID64)[440], self.TEST_ID64)

        schema_attr_exps = []
        for attr in self._schema.attributes:
            if not attr.description:
                continue

            desc = attr.description.strip()
            exp = re.escape(desc).replace("\\%s1", r"[\d-]+")

            schema_attr_exps.append(re.compile(exp))

        sim_attrs = {}
        for item in sim_inv:
            sim_attrs.setdefault(item.id, set())

            for attr in item:
                # Due to lack of contextual data, we'll have to do fuzzy matching to separate actual attrs from fluff/descriptions
                desc = attr.description.strip()
                if desc:
                    # Stop processing if we hit item set attrs, for now
                    if desc.startswith("Item Set Bonus:"):
                        break

                    # Valve for some reason insists on this being attached by the client, since they're not actually attached we skip it.
                    if desc == "Given to valuable Community Contributors":
                        continue

                    for exp in schema_attr_exps:
                        if exp.match(desc):
                            sim_attrs[item.id].add(desc)
                            break

        for item in self._inv:
            # Ignore hidden, special (for now) and date values (timestamp formatting is an eternal battle, let it not be fought on these hallowed testgrounds)
            attrs = set([attr.formatted_description for attr in item if not attr.hidden and attr.value_type not in ("date", "particle_index")])
            self.assertTrue(item.id in sim_attrs)
            self.assertEqual(attrs, sim_attrs[item.id])

class InventoryTestCase(InventoryBaseTestCase):
    def test_cell_count(self):
        self.assertLessEqual(len(list(self._inv)), self._inv.cells_total)
<|endoftext|>