<commit_before># Copyright 2020 Planet Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Functionality for interacting with the orders api"""
import asyncio
import json
import logging
import os
import time
import typing
import uuid

from .. import constants, exceptions
from ..http import Session
from ..models import Order, Orders, Request, Response, StreamingBody

BASE_URL = constants.PLANET_BASE_URL + 'compute/ops/'
STATS_PATH = 'stats/orders/v2/'
ORDERS_PATH = 'orders/v2/'
BULK_PATH = 'bulk/orders/v2/'

# Order states: https://developers.planet.com/docs/orders/ordering/#order-states
ORDERS_STATES_COMPLETE = ['success', 'partial', 'cancelled', 'failed']
ORDERS_STATES_IN_PROGRESS = ['queued', 'running']
ORDERS_STATES = ORDERS_STATES_IN_PROGRESS + ORDERS_STATES_COMPLETE

LOGGER = logging.getLogger(__name__)

class OrdersClientException(Exception):
    """Exceptions thrown by OrdersClient"""
    pass

class OrdersClient():
    """High-level asynchronous access to Planet's orders API.

    Example:
        ```python
        >>> import asyncio
        >>> from planet import Session, OrdersClient
        >>>
        >>> async def main():
        ...     auth = ('example_api_key', '')
        ...     async with Session(auth=auth) as sess:
        ...         cl = OrdersClient(sess)
        ...         # use client here
        ...
        >>> asyncio.run(main())

        ```

    """
    def __init__(
        self,
        session: Session,
        base_url: str = None
    ):
        """
        Parameters:
            session: Open session connected to server.
            base_url: The base URL to use. Defaults to production orders API
                base url.
        """
        self._session = session
        self.NUM_DOWNLOADS = 0
        self.NUM_DOWNLOADS_SKIPPED = 0

        self._base_url = base_url or BASE_URL
        if not self._base_url.endswith('/'):
            self._base_url += '/'

    @staticmethod
    def _check_order_id(oid):
        msg = f'Order id ({oid}) must be a valid UUID string.'
        if not isinstance(oid, str):
            raise OrdersClientException(msg)

        try:
            uuid.UUID(oid)
        except ValueError:
            raise OrdersClientException(msg)

    def _orders_url(self):
        return self._base_url + ORDERS_PATH

    def _stats_url(self):
        return self._base_url + STATS_PATH

    def _order_url(self, order_id):
        self._check_order_id(order_id)
        return self._orders_url() + order_id

    def _bulk_url(self):
        return self._base_url + BULK_PATH

    def _request(self, url, method, data=None, params=None, json=None):
        return Request(url, method=method, data=data, params=params, json=json)

    async def _do_request(
        self,
        request: Request
    ) -> Response:
        '''Submit a request and get response.

        Parameters:
            request: request to submit
        '''
        return await self._session.request(request)

    async def create_order(
        self,
        request: dict
    ) -> str:
        '''Create an order request.

        Example:

        ```python
        >>> import asyncio
        >>> from planet import Session, OrdersClient
        >>> from planet.order_request import build_request, product
        >>>
        >>> async def main():
        ...     image_ids = ['3949357_1454705_2020-12-01_241c']
        ...     request = build_request(
        ...         'test_order',
        ...         [product(image_ids, 'analytic', 'psorthotile')]
        ...     )
        ...     async with Session() as sess:
        ...         cl = OrdersClient(sess)
        ...         order_id = await cl.create_order(request)
        ...
        >>> asyncio.run(main())

        ```

        Parameters:
            request: order request definition

        Returns:
            The ID of the order

        Raises:
            planet.exceptions.APIException: On API error.
        '''
        url = self._orders_url()
        req = self._request(url, method='POST', json=request)

        try:
            resp = await self._do_request(req)
        except exceptions.BadQuery as ex:
            msg_json = json.loads(ex.message)

            # get first error field
            field = next(iter(msg_json['field'].keys()))
            msg = (msg_json['general'][0]['message'] + ' - ' +
                   msg_json['field'][field][0]['message'])
            raise exceptions.BadQuery(msg)

        order = Order(resp.json())
        return order

    async def get_order(
        self,
        order_id: str
    ) -> Order:
        '''Get order details by Order ID.

        Parameters:
            order_id: The ID of the order

        Returns:
            Order information

        Raises:
            OrdersClientException: If order_id is not valid UUID.
            planet.exceptions.APIException: On API error.
        '''
        url = self._order_url(order_id)

        req = self._request(url, method='GET')

        try:
            resp = await self._do_request(req)
        except exceptions.MissingResource as ex:
            msg_json = json.loads(ex.message)
            msg = msg_json['message']
            raise exceptions.MissingResource(msg)

        order = Order(resp.json())
        return order

    async def cancel_order(
        self,
        order_id: str
    ) -> Response:
        '''Cancel a queued order.

        **Note:** According to the API docs, cancel order should return the
        cancelled order details. But testing reveals that an empty response is
        returned upon success.

        Parameters:
            order_id: The ID of the order

        Returns:
            Empty response

        Raises:
            OrdersClientException: If order_id is not valid UUID.
            planet.exceptions.APIException: On API error.
        '''
        url = self._order_url(order_id)
        req = self._request(url, method='PUT')

        try:
            await self._do_request(req)
        except exceptions.Conflict as ex:
            msg = json.loads(ex.message)['message']
            raise exceptions.Conflict(msg)
        except exceptions.MissingResource as ex:
            msg = json.loads(ex.message)['message']
            raise exceptions.MissingResource(msg)

    async def cancel_orders(
        self,
        order_ids: typing.List[str] = None
    ) -> dict:
        '''Cancel queued orders in bulk.

        Parameters:
            order_ids: The IDs of the orders. If empty or None, all orders in a
                pre-running state will be cancelled.

        Returns:
            Results of the bulk cancel request

        Raises:
            planet.exceptions.APIException: On API error.
        '''
        url = self._bulk_url() + 'cancel'
        cancel_body = {}
        if order_ids:
            for oid in order_ids:
                self._check_order_id(oid)
            cancel_body['order_ids'] = order_ids

        req = self._request(url, method='POST', json=cancel_body)
        resp = await self._do_request(req)
        return resp.json()

    async def aggregated_order_stats(self) -> dict:
        '''Get aggregated counts of active orders.

        Returns:
            Aggregated order counts

        Raises:
            planet.exceptions.APIException: On API error.
        '''
        url = self._stats_url()
        req = self._request(url, method='GET')
        resp = await self._do_request(req)
        return resp.json()

    async def download_asset(
        self,
        location: str,
        order_id: str,
        filename: str = None,
        directory: str = None,
        overwrite: bool = False,
        progress_bar: bool = True
    ) -> str:
        """Download ordered asset.

        Parameters:
            location: Download location url including download token.
            order_id: The ID of the order
            filename: Custom name to assign to downloaded file.
            directory: Write to given directory instead of current directory.
            overwrite: Overwrite any existing files.
            progress_bar: Show progress bar during download.

        Returns:
            Path to downloaded file.

        Raises:
            planet.exceptions.APIException: On API error.
        """
        req = self._request(location, method='GET')

        async with self._session.stream(req) as resp:
            body = StreamingBody(resp)
            dl_path = os.path.join(directory or '.', order_id)
            # If path to download data doesn't exist, create directory
            # Consistent with https://developers.planet.com/docs/orders/delivery/
            if not os.path.isdir(dl_path):
                os.mkdir(dl_path)
            file_path = os.path.join(dl_path, filename or body.name)
            is_file_path = os.path.isfile(file_path)

            # Overwrite data or file doesn't exist in requested path
            if overwrite or not is_file_path:
                await body.write(file_path,
                                 overwrite=overwrite,
                                 progress_bar=progress_bar)
                self.NUM_DOWNLOADS += 1
            # Do not overwrite and file exists
            elif is_file_path and not overwrite:
                self.NUM_DOWNLOADS_SKIPPED += 1
                pass

        return file_path

    async def download_order(
        self,
        order_id: str,
        directory: str = None,
        overwrite: bool = False,
        progress_bar: bool = False
    ) -> typing.List[str]:
        """Download all assets in an order.

        Parameters:
            order_id: The ID of the order
            directory: Write to given directory instead of current directory.
            overwrite: Overwrite any existing files.
            progress_bar: Show progress bar during download.

        Returns:
            Paths to downloaded files.
            Number of downloaded files.
            Number of skipped downloads.

        Raises:
            planet.exceptions.APIException: On API error.
        """
        order = await self.get_order(order_id)
        locations = order.locations
        LOGGER.info(
            f'downloading {len(locations)} assets from order {order_id}'
        )
        filenames = [await self.download_asset(location,
                                               order_id,
                                               directory=directory,
                                               overwrite=overwrite,
                                               progress_bar=progress_bar)
                     for location in locations]
        return filenames, self.NUM_DOWNLOADS, self.NUM_DOWNLOADS_SKIPPED

    async def poll(
        self,
        order_id: str,
        state: str = None,
        wait: int = 10,
        verbose: bool = False
    ) -> str:
        """Poll for order status until order reaches desired state.

        Parameters:
            order_id: The ID of the order
            state: State to poll until. If multiple, use list. Defaults to
                any completed state.
            wait: Time (in seconds) between polls.
            verbose: Print current state at each poll

        Returns
            Completed state of the order

        Raises:
            planet.exceptions.APIException: On API error.
            OrdersClientException: If order_id is not valid or state is not
                supported.
        """
        completed = False

        if state:
            if state not in ORDERS_STATES:
                raise OrdersClientException(
                    f'{state} should be one of'
                    f'{ORDERS_STATES}')
            states = [state]
        else:
            states = ORDERS_STATES_COMPLETE

        while not completed:
            t = time.time()
            order = await self.get_order(order_id)
            state = order.state
            msg = f'order {order_id} state: {state}'
            LOGGER.info(msg)
            if verbose:
                print(msg)

            completed = state in states
            if not completed:
                sleep_time = max(wait-(time.time()-t), 0)
                LOGGER.info(f'sleeping {sleep_time}s')
                await asyncio.sleep(sleep_time)
        return state

    async def list_orders(
        self,
        state: str = None,
        limit: int = None,
        as_json: bool = False
    ) -> typing.Union[typing.List[Order], dict]:
        """Get all order requests.

        Parameters:
            state: Filter orders to given state.
            limit: Limit orders to given limit.
            as_json: Return orders as a json dict.

        Returns:
            User orders that match the query

        Raises:
            planet.exceptions.APIException: On API error.
        """
        url = self._orders_url()
        if state:
            self._check_state(state)
            params = {"state": state}
        else:
            params = None

        orders = await self._get_orders(url, params=params, limit=limit)

        if as_json:
            ret = [o.json async for o in orders]
        else:
            ret = [o async for o in orders]
        return ret

    async def _get_orders(self, url, params=None, limit=None):
        request = self._request(url, 'GET', params=params)

        return Orders(request, self._do_request, limit=limit)

    @staticmethod
    def _check_state(state):
        if state not in ORDERS_STATES:
            raise OrdersClientException(
                f'Order state (\'{state}\') should be one of: '
                f'{ORDERS_STATES}'
            )
<commit_msg>Back to basics: Removed number of downloads and skipped downloads. download_order() now only return all files in an order. Created new function to ensure user downloads to order ID sub-dir.<commit_after># Copyright 2020 Planet Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Functionality for interacting with the orders api"""
import asyncio
import json
import logging
import os
import time
import typing
import uuid

from .. import constants, exceptions
from ..http import Session
from ..models import Order, Orders, Request, Response, StreamingBody

BASE_URL = constants.PLANET_BASE_URL + 'compute/ops/'
STATS_PATH = 'stats/orders/v2/'
ORDERS_PATH = 'orders/v2/'
BULK_PATH = 'bulk/orders/v2/'

# Order states: https://developers.planet.com/docs/orders/ordering/#order-states
ORDERS_STATES_COMPLETE = ['success', 'partial', 'cancelled', 'failed']
ORDERS_STATES_IN_PROGRESS = ['queued', 'running']
ORDERS_STATES = ORDERS_STATES_IN_PROGRESS + ORDERS_STATES_COMPLETE

LOGGER = logging.getLogger(__name__)

class OrdersClientException(Exception):
    """Exceptions thrown by OrdersClient"""
    pass

class OrdersClient():
    """High-level asynchronous access to Planet's orders API.

    Example:
        ```python
        >>> import asyncio
        >>> from planet import Session, OrdersClient
        >>>
        >>> async def main():
        ...     auth = ('example_api_key', '')
        ...     async with Session(auth=auth) as sess:
        ...         cl = OrdersClient(sess)
        ...         # use client here
        ...
        >>> asyncio.run(main())

        ```

    """
    def __init__(
        self,
        session: Session,
        base_url: str = None
    ):
        """
        Parameters:
            session: Open session connected to server.
            base_url: The base URL to use. Defaults to production orders API
                base url.
        """
        self._session = session

        self._base_url = base_url or BASE_URL
        if not self._base_url.endswith('/'):
            self._base_url += '/'

    @staticmethod
    def _check_order_id(oid):
        msg = f'Order id ({oid}) must be a valid UUID string.'
        if not isinstance(oid, str):
            raise OrdersClientException(msg)

        try:
            uuid.UUID(oid)
        except ValueError:
            raise OrdersClientException(msg)

    def _orders_url(self):
        return self._base_url + ORDERS_PATH

    def _stats_url(self):
        return self._base_url + STATS_PATH

    def _order_url(self, order_id):
        self._check_order_id(order_id)
        return self._orders_url() + order_id

    def _bulk_url(self):
        return self._base_url + BULK_PATH

    def _request(self, url, method, data=None, params=None, json=None):
        return Request(url, method=method, data=data, params=params, json=json)

    async def _do_request(
        self,
        request: Request
    ) -> Response:
        '''Submit a request and get response.

        Parameters:
            request: request to submit
        '''
        return await self._session.request(request)

    async def create_order(
        self,
        request: dict
    ) -> str:
        '''Create an order request.

        Example:

        ```python
        >>> import asyncio
        >>> from planet import Session, OrdersClient
        >>> from planet.order_request import build_request, product
        >>>
        >>> async def main():
        ...     image_ids = ['3949357_1454705_2020-12-01_241c']
        ...     request = build_request(
        ...         'test_order',
        ...         [product(image_ids, 'analytic', 'psorthotile')]
        ...     )
        ...     async with Session() as sess:
        ...         cl = OrdersClient(sess)
        ...         order_id = await cl.create_order(request)
        ...
        >>> asyncio.run(main())

        ```

        Parameters:
            request: order request definition

        Returns:
            The ID of the order

        Raises:
            planet.exceptions.APIException: On API error.
        '''
        url = self._orders_url()
        req = self._request(url, method='POST', json=request)

        try:
            resp = await self._do_request(req)
        except exceptions.BadQuery as ex:
            msg_json = json.loads(ex.message)

            # get first error field
            field = next(iter(msg_json['field'].keys()))
            msg = (msg_json['general'][0]['message'] + ' - ' +
                   msg_json['field'][field][0]['message'])
            raise exceptions.BadQuery(msg)

        order = Order(resp.json())
        return order

    async def get_order(
        self,
        order_id: str
    ) -> Order:
        '''Get order details by Order ID.

        Parameters:
            order_id: The ID of the order

        Returns:
            Order information

        Raises:
            OrdersClientException: If order_id is not valid UUID.
            planet.exceptions.APIException: On API error.
        '''
        url = self._order_url(order_id)

        req = self._request(url, method='GET')

        try:
            resp = await self._do_request(req)
        except exceptions.MissingResource as ex:
            msg_json = json.loads(ex.message)
            msg = msg_json['message']
            raise exceptions.MissingResource(msg)

        order = Order(resp.json())
        return order

    async def cancel_order(
        self,
        order_id: str
    ) -> Response:
        '''Cancel a queued order.

        **Note:** According to the API docs, cancel order should return the
        cancelled order details. But testing reveals that an empty response is
        returned upon success.

        Parameters:
            order_id: The ID of the order

        Returns:
            Empty response

        Raises:
            OrdersClientException: If order_id is not valid UUID.
            planet.exceptions.APIException: On API error.
        '''
        url = self._order_url(order_id)
        req = self._request(url, method='PUT')

        try:
            await self._do_request(req)
        except exceptions.Conflict as ex:
            msg = json.loads(ex.message)['message']
            raise exceptions.Conflict(msg)
        except exceptions.MissingResource as ex:
            msg = json.loads(ex.message)['message']
            raise exceptions.MissingResource(msg)

    async def cancel_orders(
        self,
        order_ids: typing.List[str] = None
    ) -> dict:
        '''Cancel queued orders in bulk.

        Parameters:
            order_ids: The IDs of the orders. If empty or None, all orders in a
                pre-running state will be cancelled.

        Returns:
            Results of the bulk cancel request

        Raises:
            planet.exceptions.APIException: On API error.
        '''
        url = self._bulk_url() + 'cancel'
        cancel_body = {}
        if order_ids:
            for oid in order_ids:
                self._check_order_id(oid)
            cancel_body['order_ids'] = order_ids

        req = self._request(url, method='POST', json=cancel_body)
        resp = await self._do_request(req)
        return resp.json()

    async def aggregated_order_stats(self) -> dict:
        '''Get aggregated counts of active orders.

        Returns:
            Aggregated order counts

        Raises:
            planet.exceptions.APIException: On API error.
        '''
        url = self._stats_url()
        req = self._request(url, method='GET')
        resp = await self._do_request(req)
        return resp.json()

    async def download_asset(
        self,
        location: str,
        filename: str = None,
        directory: str = None,
        overwrite: bool = False,
        progress_bar: bool = True
    ) -> str:
        """Download ordered asset.

        Parameters:
            location: Download location url including download token.
            filename: Custom name to assign to downloaded file.
            directory: Write to given directory instead of current directory.
            overwrite: Overwrite any existing files.
            progress_bar: Show progress bar during download.

        Returns:
            All files in an order.

        Raises:
            planet.exceptions.APIException: On API error.
        """

        req = self._request(location, method='GET')

        async with self._session.stream(req) as resp:
            body = StreamingBody(resp)
            dl_path = os.path.join(directory, filename or body.name)
            await body.write(dl_path,
                            overwrite=overwrite,
                            progress_bar=progress_bar)

        return dl_path

    async def download_order(
        self,
        order_id: str,
        directory: str = None,
        overwrite: bool = False,
        progress_bar: bool = False
    ) -> list[str]:
        """Download all assets in an order.

        Parameters:
            order_id: The ID of the order
            directory: Write to given directory instead of current directory.
            overwrite: Overwrite any existing files.
            progress_bar: Show progress bar during download.

        Returns:
            Paths to downloaded files.
            Number of downloaded files.
            Number of skipped downloads.

        Raises:
            planet.exceptions.APIException: On API error.
        """
        order = await self.get_order(order_id)
        locations = order.locations
        LOGGER.info(
            f'downloading {len(locations)} assets from order {order_id}'
        )

        directory = self._set_path(order_id=order_id, directory=directory)

        filenames = [await self.download_asset(location,
                                               directory=directory,
                                               overwrite=overwrite,
                                               progress_bar=progress_bar)
                     for location in locations]

        return filenames

    async def poll(
        self,
        order_id: str,
        state: str = None,
        wait: int = 10,
        verbose: bool = False
    ) -> str:
        """Poll for order status until order reaches desired state.

        Parameters:
            order_id: The ID of the order
            state: State to poll until. If multiple, use list. Defaults to
                any completed state.
            wait: Time (in seconds) between polls.
            verbose: Print current state at each poll

        Returns
            Completed state of the order

        Raises:
            planet.exceptions.APIException: On API error.
            OrdersClientException: If order_id is not valid or state is not
                supported.
        """
        completed = False

        if state:
            if state not in ORDERS_STATES:
                raise OrdersClientException(
                    f'{state} should be one of'
                    f'{ORDERS_STATES}')
            states = [state]
        else:
            states = ORDERS_STATES_COMPLETE

        while not completed:
            t = time.time()
            order = await self.get_order(order_id)
            state = order.state
            msg = f'order {order_id} state: {state}'
            LOGGER.info(msg)
            if verbose:
                print(msg)

            completed = state in states
            if not completed:
                sleep_time = max(wait-(time.time()-t), 0)
                LOGGER.info(f'sleeping {sleep_time}s')
                await asyncio.sleep(sleep_time)
        return state

    async def list_orders(
        self,
        state: str = None,
        limit: int = None,
        as_json: bool = False
    ) -> typing.Union[typing.List[Order], dict]:
        """Get all order requests.

        Parameters:
            state: Filter orders to given state.
            limit: Limit orders to given limit.
            as_json: Return orders as a json dict.

        Returns:
            User orders that match the query

        Raises:
            planet.exceptions.APIException: On API error.
        """
        url = self._orders_url()
        if state:
            self._check_state(state)
            params = {"state": state}
        else:
            params = None

        orders = await self._get_orders(url, params=params, limit=limit)

        if as_json:
            ret = [o.json async for o in orders]
        else:
            ret = [o async for o in orders]
        return ret

    async def _get_orders(self, url, params=None, limit=None):
        request = self._request(url, 'GET', params=params)

        return Orders(request, self._do_request, limit=limit)

    @staticmethod

    def _set_path(
        order_id: str,
        directory: str
    ) -> str:
        """Ensures that the user downloads their data to a directory named by the Order ID.
           This is consistent with our order delivery layout https://developers.planet.com/docs/orders/delivery/

            Parameters:
                order_id: The ID of the order
                directory: Directory to ensure exists.

            Returns:
                Directory to downloaded files.

            Raises:
                planet.api.exceptions.APIException: On API error.
        """

        # If a directory wasn't provided, choose current working directory
        directory = directory or '.'

        if directory.split("/")[-1] != order_id:
            directory = os.path.join(directory, order_id)

        if not os.path.isdir(directory):
            os.mkdir(directory)

        return directory

    def _check_state(state):
        if state not in ORDERS_STATES:
            raise OrdersClientException(
                f'Order state (\'{state}\') should be one of: '
                f'{ORDERS_STATES}'
            )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# django-ldapdb
# Copyright (C) 2009-2010 Bolloré telecom
# See AUTHORS file for a full list of contributors.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import ldap

from django.test import TestCase

from ldapdb import connection
from examples.models import LdapUser, LdapGroup

class BaseTestCase(TestCase):
    def setUp(self):
        cursor = connection._cursor()
        for base in [LdapGroup.base_dn, LdapUser.base_dn]:
            ou = base.split(',')[0].split('=')[1]
            attrs = [('objectClass', ['top', 'organizationalUnit']), ('ou', [ou])]
            try:
                cursor.connection.add_s(base, attrs)
            except ldap.ALREADY_EXISTS:
                pass

    def tearDown(self):
        cursor = connection._cursor()
        for base in [LdapGroup.base_dn, LdapUser.base_dn]:
            results = cursor.connection.search_s(base, ldap.SCOPE_SUBTREE)
            for dn, attrs in reversed(results):
                cursor.connection.delete_s(dn)

class GroupTestCase(BaseTestCase):
    def test_create(self):
        g = LdapGroup()
        g.name = "foogroup"
        g.gid = 1000
        g.usernames = ['foouser', 'baruser']
        g.save()

    def test_get(self):
        self.test_create()

        g = LdapGroup.objects.get(name='foogroup')
        self.assertEquals(g.name, 'foogroup')
        self.assertEquals(g.gid, 1000)
        self.assertEquals(g.usernames, ['foouser', 'baruser'])

class UserTestCase(BaseTestCase):
    def test_create(self):
        u = LdapUser()
        u.first_name = "Foo"
        u.last_name = "User"
        u.full_name = "Foo User"

        u.group = 1000
        u.home_directory = "/home/foouser"
        u.uid = 1000
        u.username = "foouser"
        u.save()

    def test_get(self):
        self.test_create()

        u = LdapUser.objects.get(username='foouser')
        self.assertEquals(u.first_name, 'Foo')
        self.assertEquals(u.last_name, 'User')
        self.assertEquals(u.full_name, 'Foo User')

        self.assertEquals(u.group, 1000)
        self.assertEquals(u.home_directory, '/home/foouser')
        self.assertEquals(u.uid, 1000)
        self.assertEquals(u.username, 'foouser')
        self.assertEquals(u.photo, '')

<commit_msg>test photo attribute<commit_after># -*- coding: utf-8 -*-
#
# django-ldapdb
# Copyright (C) 2009-2010 Bolloré telecom
# See AUTHORS file for a full list of contributors.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import ldap

from django.test import TestCase

from ldapdb import connection
from examples.models import LdapUser, LdapGroup

class BaseTestCase(TestCase):
    def setUp(self):
        cursor = connection._cursor()
        for base in [LdapGroup.base_dn, LdapUser.base_dn]:
            ou = base.split(',')[0].split('=')[1]
            attrs = [('objectClass', ['top', 'organizationalUnit']), ('ou', [ou])]
            try:
                cursor.connection.add_s(base, attrs)
            except ldap.ALREADY_EXISTS:
                pass

    def tearDown(self):
        cursor = connection._cursor()
        for base in [LdapGroup.base_dn, LdapUser.base_dn]:
            results = cursor.connection.search_s(base, ldap.SCOPE_SUBTREE)
            for dn, attrs in reversed(results):
                cursor.connection.delete_s(dn)

class GroupTestCase(BaseTestCase):
    def test_create(self):
        g = LdapGroup()
        g.name = "foogroup"
        g.gid = 1000
        g.usernames = ['foouser', 'baruser']
        g.save()

    def test_get(self):
        self.test_create()

        g = LdapGroup.objects.get(name='foogroup')
        self.assertEquals(g.name, 'foogroup')
        self.assertEquals(g.gid, 1000)
        self.assertEquals(g.usernames, ['foouser', 'baruser'])

class UserTestCase(BaseTestCase):
    def test_create(self):
        u = LdapUser()
        u.first_name = "Foo"
        u.last_name = "User"
        u.full_name = "Foo User"

        u.group = 1000
        u.home_directory = "/home/foouser"
        u.uid = 1000
        u.username = "foouser"
        u.photo = '\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00H\x00H\x00\x00\xff\xfe\x00\x1cCreated with GIMP on a Mac\xff\xdb\x00C\x00\x05\x03\x04\x04\x04\x03\x05\x04\x04\x04\x05\x05\x05\x06\x07\x0c\x08\x07\x07\x07\x07\x0f\x0b\x0b\t\x0c\x11\x0f\x12\x12\x11\x0f\x11\x11\x13\x16\x1c\x17\x13\x14\x1a\x15\x11\x11\x18!\x18\x1a\x1d\x1d\x1f\x1f\x1f\x13\x17"$"\x1e$\x1c\x1e\x1f\x1e\xff\xdb\x00C\x01\x05\x05\x05\x07\x06\x07\x0e\x08\x08\x0e\x1e\x14\x11\x14\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\xff\xc0\x00\x11\x08\x00\x08\x00\x08\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x15\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\xff\xc4\x00\x19\x10\x00\x03\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02\x06\x11A\xff\xc4\x00\x14\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x11\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\x9d\xf29wU5Q\xd6\xfd\x00\x01\xff\xd9'
        u.save()

    def test_get(self):
        self.test_create()

        u = LdapUser.objects.get(username='foouser')
        self.assertEquals(u.first_name, 'Foo')
        self.assertEquals(u.last_name, 'User')
        self.assertEquals(u.full_name, 'Foo User')

        self.assertEquals(u.group, 1000)
        self.assertEquals(u.home_directory, '/home/foouser')
        self.assertEquals(u.uid, 1000)
        self.assertEquals(u.username, 'foouser')
        self.assertEquals(u.photo, '\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00H\x00H\x00\x00\xff\xfe\x00\x1cCreated with GIMP on a Mac\xff\xdb\x00C\x00\x05\x03\x04\x04\x04\x03\x05\x04\x04\x04\x05\x05\x05\x06\x07\x0c\x08\x07\x07\x07\x07\x0f\x0b\x0b\t\x0c\x11\x0f\x12\x12\x11\x0f\x11\x11\x13\x16\x1c\x17\x13\x14\x1a\x15\x11\x11\x18!\x18\x1a\x1d\x1d\x1f\x1f\x1f\x13\x17"$"\x1e$\x1c\x1e\x1f\x1e\xff\xdb\x00C\x01\x05\x05\x05\x07\x06\x07\x0e\x08\x08\x0e\x1e\x14\x11\x14\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\xff\xc0\x00\x11\x08\x00\x08\x00\x08\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x15\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\xff\xc4\x00\x19\x10\x00\x03\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02\x06\x11A\xff\xc4\x00\x14\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x11\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\x9d\xf29wU5Q\xd6\xfd\x00\x01\xff\xd9')

<|endoftext|>


<commit_before># coding=utf-8

from setuptools import setup

setup(
  name='mailchecker',
  version = "4.1.2",
  py_modules=['MailChecker'],
  url='https://github.com/FGRibreau/mailchecker',
  license='MIT',
  author='Francois-Guillaume Ribreau',
  author_email='github@fgribreau.com',
  description='Cross-language email validation. '
              'Backed by a database of thousands throwable email providers.',
  classifiers=[
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy'
  ],
)
<commit_msg>feat(python): updated to 4.1.3<commit_after># coding=utf-8

from setuptools import setup

setup(
  name='mailchecker',
  version = "4.1.3",
  py_modules=['MailChecker'],
  url='https://github.com/FGRibreau/mailchecker',
  license='MIT',
  author='Francois-Guillaume Ribreau',
  author_email='github@fgribreau.com',
  description='Cross-language email validation. '
              'Backed by a database of thousands throwable email providers.',
  classifiers=[
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy'
  ],
)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Dump out stats about all the actions that are in use in a set of replays."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import multiprocessing
import os
import signal
import sys
import threading
import time

from absl import app
from absl import flags
from future.builtins import range  # pylint: disable=redefined-builtin
import queue
import six

from pysc2 import run_configs
from pysc2.lib import features
from pysc2.lib import point
from pysc2.lib import protocol
from pysc2.lib import remote_controller

from pysc2.lib import gfile
from s2clientprotocol import common_pb2 as sc_common
from s2clientprotocol import sc2api_pb2 as sc_pb

FLAGS = flags.FLAGS
flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.")
flags.DEFINE_integer("step_mul", 8, "How many game steps per observation.")
flags.DEFINE_string("replays", None, "Path to a directory of replays.")
flags.mark_flag_as_required("replays")

size = point.Point(16, 16)
interface = sc_pb.InterfaceOptions(
    raw=True, score=False,
    feature_layer=sc_pb.SpatialCameraSetup(width=24))
size.assign_to(interface.feature_layer.resolution)
size.assign_to(interface.feature_layer.minimap_resolution)

def sorted_dict_str(d):
  return "{%s}" % ", ".join("%s: %s" % (k, d[k])
                            for k in sorted(d, key=d.get, reverse=True))

class ReplayStats(object):
  """Summary stats of the replays seen so far."""

  def __init__(self):
    self.replays = 0
    self.steps = 0
    self.camera_move = 0
    self.select_pt = 0
    self.select_rect = 0
    self.control_group = 0
    self.maps = collections.defaultdict(int)
    self.races = collections.defaultdict(int)
    self.unit_ids = collections.defaultdict(int)
    self.valid_abilities = collections.defaultdict(int)
    self.made_abilities = collections.defaultdict(int)
    self.valid_actions = collections.defaultdict(int)
    self.made_actions = collections.defaultdict(int)
    self.buffs = collections.defaultdict(int)
    self.upgrades = collections.defaultdict(int)
    self.effects = collections.defaultdict(int)
    self.crashing_replays = set()
    self.invalid_replays = set()

  def merge(self, other):
    """Merge another ReplayStats into this one."""
    def merge_dict(a, b):
      for k, v in six.iteritems(b):
        a[k] += v

    self.replays += other.replays
    self.steps += other.steps
    self.camera_move += other.camera_move
    self.select_pt += other.select_pt
    self.select_rect += other.select_rect
    self.control_group += other.control_group
    merge_dict(self.maps, other.maps)
    merge_dict(self.races, other.races)
    merge_dict(self.unit_ids, other.unit_ids)
    merge_dict(self.valid_abilities, other.valid_abilities)
    merge_dict(self.made_abilities, other.made_abilities)
    merge_dict(self.valid_actions, other.valid_actions)
    merge_dict(self.made_actions, other.made_actions)
    merge_dict(self.buffs, other.buffs)
    merge_dict(self.upgrades, other.upgrades)
    merge_dict(self.effects, other.effects)
    self.crashing_replays |= other.crashing_replays
    self.invalid_replays |= other.invalid_replays

  def __str__(self):
    len_sorted_dict = lambda s: (len(s), sorted_dict_str(s))
    len_sorted_list = lambda s: (len(s), sorted(s))
    return "\n\n".join((
        "Replays: %s, Steps total: %s" % (self.replays, self.steps),
        "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % (
            self.camera_move, self.select_pt, self.select_rect,
            self.control_group),
        "Maps: %s\n%s" % len_sorted_dict(self.maps),
        "Races: %s\n%s" % len_sorted_dict(self.races),
        "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids),
        "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities),
        "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities),
        "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions),
        "Made actions: %s\n%s" % len_sorted_dict(self.made_actions),
        "Buffs: %s\n%s" % len_sorted_dict(self.buffs),
        "Upgrades: %s\n%s" % len_sorted_dict(self.upgrades),
        "Effects: %s\n%s" % len_sorted_dict(self.effects),
        "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays),
        "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays),
    ))

class ProcessStats(object):
  """Stats for a worker process."""

  def __init__(self, proc_id):
    self.proc_id = proc_id
    self.time = time.time()
    self.stage = ""
    self.replay = ""
    self.replay_stats = ReplayStats()

  def update(self, stage):
    self.time = time.time()
    self.stage = stage

  def __str__(self):
    return ("[%2d] replay: %10s, replays: %5d, steps: %7d, game loops: %7s, "
            "last: %12s, %3d s ago" % (
                self.proc_id, self.replay, self.replay_stats.replays,
                self.replay_stats.steps,
                self.replay_stats.steps * FLAGS.step_mul, self.stage,
                time.time() - self.time))

def valid_replay(info, ping):
  """Make sure the replay isn't corrupt, and is worth looking at."""
  if (info.HasField("error") or
      info.base_build != ping.base_build or  # different game version
      info.game_duration_loops < 1000 or
      len(info.player_info) != 2):
    # Probably corrupt, or just not interesting.
    return False
  for p in info.player_info:
    if p.player_apm < 10 or p.player_mmr < 1000:
      # Low APM = player just standing around.
      # Low MMR = corrupt replay or player who is weak.
      return False
  return True

class ReplayProcessor(multiprocessing.Process):
  """A Process that pulls replays and processes them."""

  def __init__(self, proc_id, run_config, replay_queue, stats_queue):
    super(ReplayProcessor, self).__init__()
    self.stats = ProcessStats(proc_id)
    self.run_config = run_config
    self.replay_queue = replay_queue
    self.stats_queue = stats_queue

  def run(self):
    signal.signal(signal.SIGTERM, lambda a, b: sys.exit())  # Exit quietly.
    self._update_stage("spawn")
    replay_name = "none"
    while True:
      self._print("Starting up a new SC2 instance.")
      self._update_stage("launch")
      try:
        with self.run_config.start() as controller:
          self._print("SC2 Started successfully.")
          ping = controller.ping()
          for _ in range(300):
            try:
              replay_path = self.replay_queue.get()
            except queue.Empty:
              self._update_stage("done")
              self._print("Empty queue, returning")
              return
            try:
              replay_name = os.path.basename(replay_path)[:10]
              self.stats.replay = replay_name
              self._print("Got replay: %s" % replay_path)
              self._update_stage("open replay file")
              replay_data = self.run_config.replay_data(replay_path)
              self._update_stage("replay_info")
              info = controller.replay_info(replay_data)
              self._print((" Replay Info %s " % replay_name).center(60, "-"))
              self._print(info)
              self._print("-" * 60)
              if valid_replay(info, ping):
                self.stats.replay_stats.maps[info.map_name] += 1
                for player_info in info.player_info:
                  race_name = sc_common.Race.Name(
                      player_info.player_info.race_actual)
                  self.stats.replay_stats.races[race_name] += 1
                map_data = None
                if info.local_map_path:
                  self._update_stage("open map file")
                  map_data = self.run_config.map_data(info.local_map_path)
                for player_id in [1, 2]:
                  self._print("Starting %s from player %s's perspective" % (
                      replay_name, player_id))
                  self.process_replay(controller, replay_data, map_data,
                                      player_id)
              else:
                self._print("Replay is invalid.")
                self.stats.replay_stats.invalid_replays.add(replay_name)
            finally:
              self.replay_queue.task_done()
          self._update_stage("shutdown")
      except (protocol.ConnectionError, protocol.ProtocolError,
              remote_controller.RequestError):
        self.stats.replay_stats.crashing_replays.add(replay_name)
      except KeyboardInterrupt:
        return

  def _print(self, s):
    for line in str(s).strip().splitlines():
      print("[%s] %s" % (self.stats.proc_id, line))

  def _update_stage(self, stage):
    self.stats.update(stage)
    self.stats_queue.put(self.stats)

  def process_replay(self, controller, replay_data, map_data, player_id):
    """Process a single replay, updating the stats."""
    self._update_stage("start_replay")
    controller.start_replay(sc_pb.RequestStartReplay(
        replay_data=replay_data,
        map_data=map_data,
        options=interface,
        observed_player_id=player_id))

    feat = features.features_from_game_info(controller.game_info())

    self.stats.replay_stats.replays += 1
    self._update_stage("step")
    controller.step()
    while True:
      self.stats.replay_stats.steps += 1
      self._update_stage("observe")
      obs = controller.observe()

      for action in obs.actions:
        act_fl = action.action_feature_layer
        if act_fl.HasField("unit_command"):
          self.stats.replay_stats.made_abilities[
              act_fl.unit_command.ability_id] += 1
        if act_fl.HasField("camera_move"):
          self.stats.replay_stats.camera_move += 1
        if act_fl.HasField("unit_selection_point"):
          self.stats.replay_stats.select_pt += 1
        if act_fl.HasField("unit_selection_rect"):
          self.stats.replay_stats.select_rect += 1
        if action.action_ui.HasField("control_group"):
          self.stats.replay_stats.control_group += 1

        try:
          func = feat.reverse_action(action).function
        except ValueError:
          func = -1
        self.stats.replay_stats.made_actions[func] += 1

      for valid in obs.observation.abilities:
        self.stats.replay_stats.valid_abilities[valid.ability_id] += 1

      for u in obs.observation.raw_data.units:
        self.stats.replay_stats.unit_ids[u.unit_type] += 1
        for b in u.buff_ids:
          self.stats.replay_stats.buffs[b] += 1

      for u in obs.observation.raw_data.player.upgrade_ids:
        self.stats.replay_stats.upgrades[u] += 1

      for e in obs.observation.raw_data.effects:
        self.stats.replay_stats.effects[e.effect_id] += 1

      for ability_id in feat.available_actions(obs.observation):
        self.stats.replay_stats.valid_actions[ability_id] += 1

      if obs.player_result:
        break

      self._update_stage("step")
      controller.step(FLAGS.step_mul)

def stats_printer(stats_queue):
  """A thread that consumes stats_queue and prints them every 10 seconds."""
  proc_stats = [ProcessStats(i) for i in range(FLAGS.parallel)]
  print_time = start_time = time.time()
  width = 107

  running = True
  while running:
    print_time += 10

    while time.time() < print_time:
      try:
        s = stats_queue.get(True, print_time - time.time())
        if s is None:  # Signal to print and exit NOW!
          running = False
          break
        proc_stats[s.proc_id] = s
      except queue.Empty:
        pass

    replay_stats = ReplayStats()
    for s in proc_stats:
      replay_stats.merge(s.replay_stats)

    print((" Summary %0d secs " % (print_time - start_time)).center(width, "="))
    print(replay_stats)
    print(" Process stats ".center(width, "-"))
    print("\n".join(str(s) for s in proc_stats))
    print("=" * width)

def replay_queue_filler(replay_queue, replay_list):
  """A thread that fills the replay_queue with replay filenames."""
  for replay_path in replay_list:
    replay_queue.put(replay_path)

def main(unused_argv):
  """Dump stats about all the actions that are in use in a set of replays."""
  run_config = run_configs.get()

  if not gfile.Exists(FLAGS.replays):
    sys.exit("{} doesn't exist.".format(FLAGS.replays))

  stats_queue = multiprocessing.Queue()
  stats_thread = threading.Thread(target=stats_printer, args=(stats_queue,))
  stats_thread.start()
  try:
    # For some reason buffering everything into a JoinableQueue makes the
    # program not exit, so save it into a list then slowly fill it into the
    # queue in a separate thread. Grab the list synchronously so we know there
    # is work in the queue before the SC2 processes actually run, otherwise
    # The replay_queue.join below succeeds without doing any work, and exits.
    print("Getting replay list:", FLAGS.replays)
    replay_list = sorted(run_config.replay_paths(FLAGS.replays))
    print(len(replay_list), "replays found.\n")
    replay_queue = multiprocessing.JoinableQueue(FLAGS.parallel * 10)
    replay_queue_thread = threading.Thread(target=replay_queue_filler,
                                           args=(replay_queue, replay_list))
    replay_queue_thread.daemon = True
    replay_queue_thread.start()

    for i in range(FLAGS.parallel):
      p = ReplayProcessor(i, run_config, replay_queue, stats_queue)
      p.daemon = True
      p.start()
      time.sleep(1)  # Stagger startups, otherwise they seem to conflict somehow

    replay_queue.join()  # Wait for the queue to empty.
  except KeyboardInterrupt:
    print("Caught KeyboardInterrupt, exiting.")
  finally:
    stats_queue.put(None)  # Tell the stats_thread to print and exit.
    stats_thread.join()

if __name__ == "__main__":
  app.run(main)
<commit_msg>Faster startup time if you're only checking a few replays.<commit_after>#!/usr/bin/python
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Dump out stats about all the actions that are in use in a set of replays."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import multiprocessing
import os
import signal
import sys
import threading
import time

from absl import app
from absl import flags
from future.builtins import range  # pylint: disable=redefined-builtin
import queue
import six

from pysc2 import run_configs
from pysc2.lib import features
from pysc2.lib import point
from pysc2.lib import protocol
from pysc2.lib import remote_controller

from pysc2.lib import gfile
from s2clientprotocol import common_pb2 as sc_common
from s2clientprotocol import sc2api_pb2 as sc_pb

FLAGS = flags.FLAGS
flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.")
flags.DEFINE_integer("step_mul", 8, "How many game steps per observation.")
flags.DEFINE_string("replays", None, "Path to a directory of replays.")
flags.mark_flag_as_required("replays")

size = point.Point(16, 16)
interface = sc_pb.InterfaceOptions(
    raw=True, score=False,
    feature_layer=sc_pb.SpatialCameraSetup(width=24))
size.assign_to(interface.feature_layer.resolution)
size.assign_to(interface.feature_layer.minimap_resolution)

def sorted_dict_str(d):
  return "{%s}" % ", ".join("%s: %s" % (k, d[k])
                            for k in sorted(d, key=d.get, reverse=True))

class ReplayStats(object):
  """Summary stats of the replays seen so far."""

  def __init__(self):
    self.replays = 0
    self.steps = 0
    self.camera_move = 0
    self.select_pt = 0
    self.select_rect = 0
    self.control_group = 0
    self.maps = collections.defaultdict(int)
    self.races = collections.defaultdict(int)
    self.unit_ids = collections.defaultdict(int)
    self.valid_abilities = collections.defaultdict(int)
    self.made_abilities = collections.defaultdict(int)
    self.valid_actions = collections.defaultdict(int)
    self.made_actions = collections.defaultdict(int)
    self.buffs = collections.defaultdict(int)
    self.upgrades = collections.defaultdict(int)
    self.effects = collections.defaultdict(int)
    self.crashing_replays = set()
    self.invalid_replays = set()

  def merge(self, other):
    """Merge another ReplayStats into this one."""
    def merge_dict(a, b):
      for k, v in six.iteritems(b):
        a[k] += v

    self.replays += other.replays
    self.steps += other.steps
    self.camera_move += other.camera_move
    self.select_pt += other.select_pt
    self.select_rect += other.select_rect
    self.control_group += other.control_group
    merge_dict(self.maps, other.maps)
    merge_dict(self.races, other.races)
    merge_dict(self.unit_ids, other.unit_ids)
    merge_dict(self.valid_abilities, other.valid_abilities)
    merge_dict(self.made_abilities, other.made_abilities)
    merge_dict(self.valid_actions, other.valid_actions)
    merge_dict(self.made_actions, other.made_actions)
    merge_dict(self.buffs, other.buffs)
    merge_dict(self.upgrades, other.upgrades)
    merge_dict(self.effects, other.effects)
    self.crashing_replays |= other.crashing_replays
    self.invalid_replays |= other.invalid_replays

  def __str__(self):
    len_sorted_dict = lambda s: (len(s), sorted_dict_str(s))
    len_sorted_list = lambda s: (len(s), sorted(s))
    return "\n\n".join((
        "Replays: %s, Steps total: %s" % (self.replays, self.steps),
        "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % (
            self.camera_move, self.select_pt, self.select_rect,
            self.control_group),
        "Maps: %s\n%s" % len_sorted_dict(self.maps),
        "Races: %s\n%s" % len_sorted_dict(self.races),
        "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids),
        "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities),
        "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities),
        "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions),
        "Made actions: %s\n%s" % len_sorted_dict(self.made_actions),
        "Buffs: %s\n%s" % len_sorted_dict(self.buffs),
        "Upgrades: %s\n%s" % len_sorted_dict(self.upgrades),
        "Effects: %s\n%s" % len_sorted_dict(self.effects),
        "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays),
        "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays),
    ))

class ProcessStats(object):
  """Stats for a worker process."""

  def __init__(self, proc_id):
    self.proc_id = proc_id
    self.time = time.time()
    self.stage = ""
    self.replay = ""
    self.replay_stats = ReplayStats()

  def update(self, stage):
    self.time = time.time()
    self.stage = stage

  def __str__(self):
    return ("[%2d] replay: %10s, replays: %5d, steps: %7d, game loops: %7s, "
            "last: %12s, %3d s ago" % (
                self.proc_id, self.replay, self.replay_stats.replays,
                self.replay_stats.steps,
                self.replay_stats.steps * FLAGS.step_mul, self.stage,
                time.time() - self.time))

def valid_replay(info, ping):
  """Make sure the replay isn't corrupt, and is worth looking at."""
  if (info.HasField("error") or
      info.base_build != ping.base_build or  # different game version
      info.game_duration_loops < 1000 or
      len(info.player_info) != 2):
    # Probably corrupt, or just not interesting.
    return False
  for p in info.player_info:
    if p.player_apm < 10 or p.player_mmr < 1000:
      # Low APM = player just standing around.
      # Low MMR = corrupt replay or player who is weak.
      return False
  return True

class ReplayProcessor(multiprocessing.Process):
  """A Process that pulls replays and processes them."""

  def __init__(self, proc_id, run_config, replay_queue, stats_queue):
    super(ReplayProcessor, self).__init__()
    self.stats = ProcessStats(proc_id)
    self.run_config = run_config
    self.replay_queue = replay_queue
    self.stats_queue = stats_queue

  def run(self):
    signal.signal(signal.SIGTERM, lambda a, b: sys.exit())  # Exit quietly.
    self._update_stage("spawn")
    replay_name = "none"
    while True:
      self._print("Starting up a new SC2 instance.")
      self._update_stage("launch")
      try:
        with self.run_config.start() as controller:
          self._print("SC2 Started successfully.")
          ping = controller.ping()
          for _ in range(300):
            try:
              replay_path = self.replay_queue.get()
            except queue.Empty:
              self._update_stage("done")
              self._print("Empty queue, returning")
              return
            try:
              replay_name = os.path.basename(replay_path)[:10]
              self.stats.replay = replay_name
              self._print("Got replay: %s" % replay_path)
              self._update_stage("open replay file")
              replay_data = self.run_config.replay_data(replay_path)
              self._update_stage("replay_info")
              info = controller.replay_info(replay_data)
              self._print((" Replay Info %s " % replay_name).center(60, "-"))
              self._print(info)
              self._print("-" * 60)
              if valid_replay(info, ping):
                self.stats.replay_stats.maps[info.map_name] += 1
                for player_info in info.player_info:
                  race_name = sc_common.Race.Name(
                      player_info.player_info.race_actual)
                  self.stats.replay_stats.races[race_name] += 1
                map_data = None
                if info.local_map_path:
                  self._update_stage("open map file")
                  map_data = self.run_config.map_data(info.local_map_path)
                for player_id in [1, 2]:
                  self._print("Starting %s from player %s's perspective" % (
                      replay_name, player_id))
                  self.process_replay(controller, replay_data, map_data,
                                      player_id)
              else:
                self._print("Replay is invalid.")
                self.stats.replay_stats.invalid_replays.add(replay_name)
            finally:
              self.replay_queue.task_done()
          self._update_stage("shutdown")
      except (protocol.ConnectionError, protocol.ProtocolError,
              remote_controller.RequestError):
        self.stats.replay_stats.crashing_replays.add(replay_name)
      except KeyboardInterrupt:
        return

  def _print(self, s):
    for line in str(s).strip().splitlines():
      print("[%s] %s" % (self.stats.proc_id, line))

  def _update_stage(self, stage):
    self.stats.update(stage)
    self.stats_queue.put(self.stats)

  def process_replay(self, controller, replay_data, map_data, player_id):
    """Process a single replay, updating the stats."""
    self._update_stage("start_replay")
    controller.start_replay(sc_pb.RequestStartReplay(
        replay_data=replay_data,
        map_data=map_data,
        options=interface,
        observed_player_id=player_id))

    feat = features.features_from_game_info(controller.game_info())

    self.stats.replay_stats.replays += 1
    self._update_stage("step")
    controller.step()
    while True:
      self.stats.replay_stats.steps += 1
      self._update_stage("observe")
      obs = controller.observe()

      for action in obs.actions:
        act_fl = action.action_feature_layer
        if act_fl.HasField("unit_command"):
          self.stats.replay_stats.made_abilities[
              act_fl.unit_command.ability_id] += 1
        if act_fl.HasField("camera_move"):
          self.stats.replay_stats.camera_move += 1
        if act_fl.HasField("unit_selection_point"):
          self.stats.replay_stats.select_pt += 1
        if act_fl.HasField("unit_selection_rect"):
          self.stats.replay_stats.select_rect += 1
        if action.action_ui.HasField("control_group"):
          self.stats.replay_stats.control_group += 1

        try:
          func = feat.reverse_action(action).function
        except ValueError:
          func = -1
        self.stats.replay_stats.made_actions[func] += 1

      for valid in obs.observation.abilities:
        self.stats.replay_stats.valid_abilities[valid.ability_id] += 1

      for u in obs.observation.raw_data.units:
        self.stats.replay_stats.unit_ids[u.unit_type] += 1
        for b in u.buff_ids:
          self.stats.replay_stats.buffs[b] += 1

      for u in obs.observation.raw_data.player.upgrade_ids:
        self.stats.replay_stats.upgrades[u] += 1

      for e in obs.observation.raw_data.effects:
        self.stats.replay_stats.effects[e.effect_id] += 1

      for ability_id in feat.available_actions(obs.observation):
        self.stats.replay_stats.valid_actions[ability_id] += 1

      if obs.player_result:
        break

      self._update_stage("step")
      controller.step(FLAGS.step_mul)

def stats_printer(stats_queue):
  """A thread that consumes stats_queue and prints them every 10 seconds."""
  proc_stats = [ProcessStats(i) for i in range(FLAGS.parallel)]
  print_time = start_time = time.time()
  width = 107

  running = True
  while running:
    print_time += 10

    while time.time() < print_time:
      try:
        s = stats_queue.get(True, print_time - time.time())
        if s is None:  # Signal to print and exit NOW!
          running = False
          break
        proc_stats[s.proc_id] = s
      except queue.Empty:
        pass

    replay_stats = ReplayStats()
    for s in proc_stats:
      replay_stats.merge(s.replay_stats)

    print((" Summary %0d secs " % (print_time - start_time)).center(width, "="))
    print(replay_stats)
    print(" Process stats ".center(width, "-"))
    print("\n".join(str(s) for s in proc_stats))
    print("=" * width)

def replay_queue_filler(replay_queue, replay_list):
  """A thread that fills the replay_queue with replay filenames."""
  for replay_path in replay_list:
    replay_queue.put(replay_path)

def main(unused_argv):
  """Dump stats about all the actions that are in use in a set of replays."""
  run_config = run_configs.get()

  if not gfile.Exists(FLAGS.replays):
    sys.exit("{} doesn't exist.".format(FLAGS.replays))

  stats_queue = multiprocessing.Queue()
  stats_thread = threading.Thread(target=stats_printer, args=(stats_queue,))
  stats_thread.start()
  try:
    # For some reason buffering everything into a JoinableQueue makes the
    # program not exit, so save it into a list then slowly fill it into the
    # queue in a separate thread. Grab the list synchronously so we know there
    # is work in the queue before the SC2 processes actually run, otherwise
    # The replay_queue.join below succeeds without doing any work, and exits.
    print("Getting replay list:", FLAGS.replays)
    replay_list = sorted(run_config.replay_paths(FLAGS.replays))
    print(len(replay_list), "replays found.\n")
    replay_queue = multiprocessing.JoinableQueue(FLAGS.parallel * 10)
    replay_queue_thread = threading.Thread(target=replay_queue_filler,
                                           args=(replay_queue, replay_list))
    replay_queue_thread.daemon = True
    replay_queue_thread.start()

    for i in range(min(len(replay_list), FLAGS.parallel)):
      p = ReplayProcessor(i, run_config, replay_queue, stats_queue)
      p.daemon = True
      p.start()
      time.sleep(1)  # Stagger startups, otherwise they seem to conflict somehow

    replay_queue.join()  # Wait for the queue to empty.
  except KeyboardInterrupt:
    print("Caught KeyboardInterrupt, exiting.")
  finally:
    stats_queue.put(None)  # Tell the stats_thread to print and exit.
    stats_thread.join()

if __name__ == "__main__":
  app.run(main)
<|endoftext|>


<commit_before>from collections import deque
from typing import Any, Iterable, Dict

from plenum.common.constants import BATCH, OP_FIELD_NAME
from plenum.common.metrics_collector import NullMetricsCollector, MetricsName, measure_time
from plenum.common.prepare_batch import split_messages_on_batches
from stp_core.common.constants import CONNECTION_PREFIX
from stp_core.crypto.signer import Signer
from stp_core.common.log import getlogger
from plenum.common.types import f
from plenum.common.messages.node_messages import Batch
from plenum.common.message_processor import MessageProcessor
from stp_core.validators.message_length_validator import MessageLenValidator
from stp_core.common.config.util import getConfig

logger = getlogger()

class Batched(MessageProcessor):
    """
    A mixin to allow batching of requests to be send to remotes.
    Assumes a Stack (ZStack or RStack) is mixed
    """

    def __init__(self, config=None, metrics=NullMetricsCollector()):
        """
        :param self: 'NodeStacked'
        :param config: 'stp config'
        """
        self.outBoxes = {}  # type: Dict[int, deque]
        self.stp_config = config or getConfig()
        self.msg_len_val = MessageLenValidator(self.stp_config.MSG_LEN_LIMIT)
        self.metrics = metrics

    def _enqueue(self, msg: Any, rid: int, signer: Signer) -> None:
        """
        Enqueue the message into the remote's queue.

        :param msg: the message to enqueue
        :param rid: the id of the remote node
        """
        if rid not in self.outBoxes:
            self.outBoxes[rid] = deque()
        self.outBoxes[rid].append(msg)

    def _enqueueIntoAllRemotes(self, msg: Any, signer: Signer) -> None:
        """
        Enqueue the specified message into all the remotes in the nodestack.

        :param msg: the message to enqueue
        """
        for rid in self.remotes.keys():
            self._enqueue(msg, rid, signer)

    def send(self,
             msg: Any, *
             rids: Iterable[int],
             signer: Signer = None,
             message_splitter=None) -> None:
        """
        Enqueue the given message into the outBoxes of the specified remotes
        or into the outBoxes of all the remotes if rids is None

        :param msg: the message to enqueue
        :param rids: ids of the remotes to whose outBoxes
            this message must be enqueued
        :param message_splitter: callable that splits msg on
            two smaller messages
        """
        # Signing (if required) and serializing before enqueueing otherwise
        # each call to `_enqueue` will have to sign it and `transmit` will try
        # to serialize it which is waste of resources
        message_parts, err_msg = \
            self.prepare_for_sending(msg, signer, message_splitter)

        # TODO: returning breaks contract of super class
        if err_msg is not None:
            return False, err_msg

        if rids:
            for r in rids:
                for part in message_parts:
                    self._enqueue(part, r, signer)
        else:
            for part in message_parts:
                self._enqueueIntoAllRemotes(part, signer)
        return True, None

    def flushOutBoxes(self) -> None:
        """
        Clear the outBoxes and transmit batched messages to remotes.
        """
        removedRemotes = []
        for rid, msgs in self.outBoxes.items():
            try:
                dest = self.remotes[rid].name
            except KeyError:
                removedRemotes.append(rid)
                continue
            if msgs:
                if self._should_batch(msgs):
                    logger.trace(
                        "{} batching {} msgs to {} into fewer transmissions".
                        format(self, len(msgs), dest))
                    logger.trace("    messages: {}".format(msgs))
                    batches = split_messages_on_batches(list(msgs),
                                                        self._make_batch,
                                                        self._test_batch_len,
                                                        )
                    msgs.clear()
                    if batches:
                        for batch, size in batches:
                            logger.trace("{} sending payload to {}: {}".format(
                                self, dest, batch))
                            self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, size)
                            # Setting timeout to never expire
                            self.transmit(
                                batch,
                                rid,
                                timeout=self.messageTimeout,
                                serialized=True)
                    else:
                        logger.error("{} cannot create batch(es) for {}".format(self, dest))
                else:
                    while msgs:
                        msg = msgs.popleft()
                        logger.trace(
                            "{} sending msg {} to {}".format(self, msg, dest))
                        self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, 1)
                        # Setting timeout to never expire
                        self.transmit(msg, rid, timeout=self.messageTimeout,
                                      serialized=True)

        for rid in removedRemotes:
            logger.warning("{}{} has removed rid {}"
                           .format(CONNECTION_PREFIX, self, rid),
                           extra={"cli": False})
            msgs = self.outBoxes[rid]
            if msgs:
                self.discard(msgs,
                             "{}rid {} no longer available"
                             .format(CONNECTION_PREFIX, rid),
                             logMethod=logger.debug)
            del self.outBoxes[rid]

    def _make_batch(self, msgs):
        if len(msgs) > 1:
            batch = Batch(msgs, None)
            serialized_batch = self.sign_and_serialize(batch)
        else:
            serialized_batch = msgs[0]
        return serialized_batch

    def _test_batch_len(self, batch_len):
        return self.msg_len_val.is_len_less_than_limit(batch_len)

    def doProcessReceived(self, msg, frm, ident):
        if OP_FIELD_NAME in msg and msg[OP_FIELD_NAME] == BATCH:
            if f.MSGS.nm in msg and isinstance(msg[f.MSGS.nm], list):
                # Removing ping and pong messages from Batch
                relevantMsgs = []
                for m in msg[f.MSGS.nm]:
                    r = self.handlePingPong(m, frm, ident)
                    if not r:
                        relevantMsgs.append(m)

                if not relevantMsgs:
                    return None
                msg[f.MSGS.nm] = relevantMsgs
        return msg

    def prepare_for_sending(self, msg, signer,
                            message_splitter=lambda x: None):
        large_msg_parts = [msg]
        fine_msg_parts = []
        while len(large_msg_parts):
            part = large_msg_parts.pop()
            part_bytes = self.sign_and_serialize(part, signer)
            if self.msg_len_val.is_len_less_than_limit(len(part_bytes)):
                fine_msg_parts.append(part_bytes)
                continue
            if message_splitter is not None:
                smaller_parts = message_splitter(part)
                if smaller_parts is not None:
                    large_msg_parts.extend(smaller_parts)
                    continue

            large_msg_parts.append(part)
            break

        if len(large_msg_parts):
            return None, "message is too large and cannot be split"

        return fine_msg_parts, None

    def sign_and_serialize(self, msg, signer=None):
        payload = self.prepForSending(msg, signer)
        msg_bytes = self.serializeMsg(payload)
        return msg_bytes

    def _should_batch(self, msgs):
        return len(msgs) > 1
<commit_msg>INDY-209: Additional formatting<commit_after>from binascii import hexlify
from collections import deque
from typing import Any, Iterable, Dict

from plenum.common.util import hexToFriendly
from zmq.utils import z85

from plenum.common.constants import BATCH, OP_FIELD_NAME
from plenum.common.metrics_collector import NullMetricsCollector, MetricsName, measure_time
from plenum.common.prepare_batch import split_messages_on_batches
from stp_core.common.constants import CONNECTION_PREFIX
from stp_core.crypto.signer import Signer
from stp_core.common.log import getlogger
from plenum.common.types import f
from plenum.common.messages.node_messages import Batch
from plenum.common.message_processor import MessageProcessor
from stp_core.validators.message_length_validator import MessageLenValidator
from stp_core.common.config.util import getConfig

logger = getlogger()

class Batched(MessageProcessor):
    """
    A mixin to allow batching of requests to be send to remotes.
    Assumes a Stack (ZStack or RStack) is mixed
    """

    def __init__(self, config=None, metrics=NullMetricsCollector()):
        """
        :param self: 'NodeStacked'
        :param config: 'stp config'
        """
        self.outBoxes = {}  # type: Dict[int, deque]
        self.stp_config = config or getConfig()
        self.msg_len_val = MessageLenValidator(self.stp_config.MSG_LEN_LIMIT)
        self.metrics = metrics

    def _enqueue(self, msg: Any, rid: int, signer: Signer) -> None:
        """
        Enqueue the message into the remote's queue.

        :param msg: the message to enqueue
        :param rid: the id of the remote node
        """
        if rid not in self.outBoxes:
            self.outBoxes[rid] = deque()
        self.outBoxes[rid].append(msg)

    def _enqueueIntoAllRemotes(self, msg: Any, signer: Signer) -> None:
        """
        Enqueue the specified message into all the remotes in the nodestack.

        :param msg: the message to enqueue
        """
        for rid in self.remotes.keys():
            self._enqueue(msg, rid, signer)

    def send(self,
             msg: Any, *
             rids: Iterable[int],
             signer: Signer = None,
             message_splitter=None) -> None:
        """
        Enqueue the given message into the outBoxes of the specified remotes
        or into the outBoxes of all the remotes if rids is None

        :param msg: the message to enqueue
        :param rids: ids of the remotes to whose outBoxes
            this message must be enqueued
        :param message_splitter: callable that splits msg on
            two smaller messages
        """
        # Signing (if required) and serializing before enqueueing otherwise
        # each call to `_enqueue` will have to sign it and `transmit` will try
        # to serialize it which is waste of resources
        message_parts, err_msg = \
            self.prepare_for_sending(msg, signer, message_splitter)

        # TODO: returning breaks contract of super class
        if err_msg is not None:
            return False, err_msg

        if rids:
            for r in rids:
                for part in message_parts:
                    self._enqueue(part, r, signer)
        else:
            for part in message_parts:
                self._enqueueIntoAllRemotes(part, signer)
        return True, None

    def flushOutBoxes(self) -> None:
        """
        Clear the outBoxes and transmit batched messages to remotes.
        """
        removedRemotes = []
        for rid, msgs in self.outBoxes.items():
            try:
                dest = self.remotes[rid].name
            except KeyError:
                removedRemotes.append(rid)
                continue
            if msgs:
                if self._should_batch(msgs):
                    logger.trace(
                        "{} batching {} msgs to {} into fewer transmissions".
                        format(self, len(msgs), dest))
                    logger.trace("    messages: {}".format(msgs))
                    batches = split_messages_on_batches(list(msgs),
                                                        self._make_batch,
                                                        self._test_batch_len,
                                                        )
                    msgs.clear()
                    if batches:
                        for batch, size in batches:
                            logger.trace("{} sending payload to {}: {}".format(
                                self, dest, batch))
                            self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, size)
                            # Setting timeout to never expire
                            self.transmit(
                                batch,
                                rid,
                                timeout=self.messageTimeout,
                                serialized=True)
                    else:
                        logger.error("{} cannot create batch(es) for {}".format(self, dest))
                else:
                    while msgs:
                        msg = msgs.popleft()
                        logger.trace(
                            "{} sending msg {} to {}".format(self, msg, dest))
                        self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, 1)
                        # Setting timeout to never expire
                        self.transmit(msg, rid, timeout=self.messageTimeout,
                                      serialized=True)

        for rid in removedRemotes:
            if not isinstance(rid, str):
                rid = hexToFriendly(hexlify(z85.decode(rid)).decode())
            logger.warning("{}{} has removed rid {}"
                           .format(CONNECTION_PREFIX, self, rid),
                           extra={"cli": False})
            msgs = self.outBoxes[rid]
            if msgs:
                self.discard(msgs,
                             "{}rid {} no longer available"
                             .format(CONNECTION_PREFIX, rid),
                             logMethod=logger.debug)
            del self.outBoxes[rid]

    def _make_batch(self, msgs):
        if len(msgs) > 1:
            batch = Batch(msgs, None)
            serialized_batch = self.sign_and_serialize(batch)
        else:
            serialized_batch = msgs[0]
        return serialized_batch

    def _test_batch_len(self, batch_len):
        return self.msg_len_val.is_len_less_than_limit(batch_len)

    def doProcessReceived(self, msg, frm, ident):
        if OP_FIELD_NAME in msg and msg[OP_FIELD_NAME] == BATCH:
            if f.MSGS.nm in msg and isinstance(msg[f.MSGS.nm], list):
                # Removing ping and pong messages from Batch
                relevantMsgs = []
                for m in msg[f.MSGS.nm]:
                    r = self.handlePingPong(m, frm, ident)
                    if not r:
                        relevantMsgs.append(m)

                if not relevantMsgs:
                    return None
                msg[f.MSGS.nm] = relevantMsgs
        return msg

    def prepare_for_sending(self, msg, signer,
                            message_splitter=lambda x: None):
        large_msg_parts = [msg]
        fine_msg_parts = []
        while len(large_msg_parts):
            part = large_msg_parts.pop()
            part_bytes = self.sign_and_serialize(part, signer)
            if self.msg_len_val.is_len_less_than_limit(len(part_bytes)):
                fine_msg_parts.append(part_bytes)
                continue
            if message_splitter is not None:
                smaller_parts = message_splitter(part)
                if smaller_parts is not None:
                    large_msg_parts.extend(smaller_parts)
                    continue

            large_msg_parts.append(part)
            break

        if len(large_msg_parts):
            return None, "message is too large and cannot be split"

        return fine_msg_parts, None

    def sign_and_serialize(self, msg, signer=None):
        payload = self.prepForSending(msg, signer)
        msg_bytes = self.serializeMsg(payload)
        return msg_bytes

    def _should_batch(self, msgs):
        return len(msgs) > 1
<|endoftext|>