<commit_before>"""Abstractions over S3's upload/download operations.

This module provides high level abstractions for efficient
uploads/downloads.  It handles several things for the user:

* Automatically switching to multipart transfers when
  a file is over a specific size threshold
* Uploading/downloading a file in parallel
* Throttling based on max bandwidth
* Progress callbacks to monitor transfers
* Retries.  While botocore handles retries for streaming uploads,
  it is not possible for it to handle retries for streaming
  downloads.  This module handles retries for both cases so
  you don't need to implement any retry logic yourself.

This module has a reasonable set of defaults.  It also allows you
to configure many aspects of the transfer process including:

* Multipart threshold size
* Max parallel downloads
* Max bandwidth
* Socket timeouts
* Retry amounts

There is no support for s3->s3 multipart copies at this
time.

Usage
=====

The simplest way to use this module is:

.. code-block:: python

    client = boto3.client('s3', 'us-west-2')
    transfer = S3Transfer(client)
    # Upload /tmp/myfile to s3://bucket/key
    transfer.upload_file('/tmp/myfile', 'bucket', 'key')

    # Download s3://bucket/key to /tmp/myfile
    transfer.download_file('bucket', 'key', '/tmp/myfile')

The ``upload_file`` and ``download_file`` methods also accept
``**kwargs``, which will be forwarded through to the corresponding
client operation.  Here are a few examples using ``upload_file``::

    # Making the object public
    transfer.upload_file('/tmp/myfile', 'bucket', 'key', ACL='public-read')

    # Setting metadata
    transfer.upload_file('/tmp/myfile', 'bucket', 'key',
                         Metadata={'a': 'b', 'c': 'd'})

    # Setting content type
    transfer.upload_file('/tmp/myfile.json', 'bucket', 'key',
                         ContentType="application/json")

The ``S3Transfer`` clas also supports progress callbacks so you can
provide transfer progress to users.  Both the ``upload_file`` and
``download_file`` methods take an optional ``callback`` parameter.
Here's an example of how to print a simple progress percentage
to the user:

.. code-block:: python

    class ProgressPercentage(object):
        def __init__(self, filename):
            self._filename = filename
            self._size = float(os.path.getsize(filename))
            self._seen_so_far = 0
            self._lock = threading.Lock()

        def __call__(self, filename, bytes_amount):
            # To simplify we'll assume this is hooked up
            # to a single filename.
            with self._lock:
                self._seen_so_far += bytes_amount
                percentage = (self._seen_so_far / self._size) * 100
                sys.stdout.write(
                    "\r%s  %s / %s  (%.2f%%)" % (filename, self._seen_so_far,
                                                 self._size, percentage))
                sys.stdout.flush()

    transfer = S3Transfer(boto3.client('s3', 'us-west-2'))
    # Upload /tmp/myfile to s3://bucket/key and print upload progress.
    transfer.upload_file('/tmp/myfile', 'bucket', 'key',
                         callback=ProgressPercentage('/tmp/myfile'))

You can also provide an TransferConfig object to the S3Transfer
object that gives you more fine grained control over the
transfer.  For example:

.. code-block:: python

    client = boto3.client('s3', 'us-west-2')
    config = TransferConfig(
        multipart_threshold=8 * 1024 * 1024,
        max_concurrency=10,
        max_retries=10,
        socket_timeout=120,
    )
    transfer = S3Transfer(client, config)
    transfer.upload_file('/tmp/foo', 'bucket', 'key')

"""
import os
import math
import threading
import functools
import logging
import socket
from concurrent import futures

from botocore.vendored.requests.packages.urllib3.exceptions import \
    ReadTimeoutError
from botocore.exceptions import IncompleteReadError

from boto3.exceptions import RetriesExceededError

logger = logging.getLogger(__name__)
MB = 1024 * 1024

class ReadFileChunk(object):
    def __init__(self, fileobj, start_byte, chunk_size, full_file_size,
                 callback=None):
        """

        Given a file object shown below:

            |___________________________________________________|
            0          |                 |                 full_file_size
                       |----chunk_size---|
                 start_byte

        :type fileobj: file
        :param fileobj: File like object

        :type start_byte: int
        :param start_byte: The first byte from which to start reading.

        :type chunk_size: int
        :param chunk_size: The max chunk size to read.  Trying to read
            pass the end of the chunk size will behave like you've
            reached the end of the file.

        :type full_file_size: int
        :param full_file_size: The entire content length associated
            with ``fileobj``.

        :type callback: function(amount_read)
        :param callback: Called whenever data is read from this object.

        """
        self._fileobj = fileobj
        self._start_byte = start_byte
        self._size = self._calculate_file_size(
            self._fileobj, requested_size=chunk_size,
            start_byte=start_byte, actual_file_size=full_file_size)
        self._fileobj.seek(self._start_byte)
        self._amount_read = 0
        self._callback = callback

    @classmethod
    def from_filename(cls, filename, start_byte, chunk_size, callback=None):
        """Convenience factory function to create from a filename."""
        f = open(filename, 'rb')
        file_size = os.fstat(f.fileno()).st_size
        return cls(f, start_byte, chunk_size, file_size, callback)

    def _calculate_file_size(self, fileobj, requested_size, start_byte,
                             actual_file_size):
        max_chunk_size = actual_file_size - start_byte
        return min(max_chunk_size, requested_size)

    def read(self, amount=None):
        if amount is None:
            amount_to_read = self._size - self._amount_read
        else:
            amount_to_read = min(self._size - self._amount_read, amount)
        data = self._fileobj.read(amount_to_read)
        self._amount_read += len(data)
        if self._callback is not None:
            self._callback(len(data))
        return data

    def seek(self, where):
        self._fileobj.seek(self._start_byte + where)
        self._amount_read = where

    def close(self):
        self._fileobj.close()

    def tell(self):
        return self._amount_read

    def __len__(self):
        # __len__ is defined because requests will try to determine the length
        # of the stream to set a content length.  In the normal case
        # of the file it will just stat the file, but we need to change that
        # behavior.  By providing a __len__, requests will use that instead
        # of stat'ing the file.
        return self._size

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def __iter__(self):
        # This is a workaround for http://bugs.python.org/issue17575
        # Basically httplib will try to iterate over the contents, even
        # if its a file like object.  This wasn't noticed because we've
        # already exhausted the stream so iterating over the file immediately
        # steps, which is what we're simulating here.
        return iter([])

class StreamReaderProgress(object):
    """Wrapper for a read only stream that adds progress callbacks."""
    def __init__(self, stream, callback=None):
        self._stream = stream
        self._callback = callback

    def read(self, *args, **kwargs):
        value = self._stream.read(*args, **kwargs)
        if self._callback is not None:
            self._callback(len(value))
        return value

class ThreadSafeWriter(object):
    def __init__(self, write_stream):
        self._write_stream = write_stream
        self._lock = threading.Lock()

    def pwrite(self, data, offset):
        with self._lock:
            self._write_stream.seek(offset)
            self._write_stream.write(data)

    def close(self):
        return self._write_stream.close()

class OSUtils(object):
    def get_file_size(self, filename):
        return os.path.getsize(filename)

    def open_file_chunk_reader(self, filename, start_byte, size, callback):
        return ReadFileChunk.from_filename(filename, start_byte,
                                           size, callback)

    def open(self, filename, mode):
        return open(filename, mode)

    def wrap_stream_with_callback(self, stream, callback):
        return StreamReaderProgress(stream, callback)

    def wrap_thread_safe_writer(self, stream):
        return ThreadSafeWriter(stream)

class MultipartUploader(object):
    def __init__(self, client, config, osutil,
                 executor_cls=futures.ThreadPoolExecutor):
        self._client = client
        self._config = config
        self._os = osutil
        self._executor_cls = executor_cls

    def upload_file(self, filename, bucket, key, callback, extra_args):
        response = self._client.create_multipart_upload(Bucket=bucket,
                                                        Key=key, **extra_args)
        upload_id = response['UploadId']
        parts = []
        part_size = self._config.multipart_chunksize
        num_parts = int(
            math.ceil(self._os.get_file_size(filename) / float(part_size)))
        max_workers = self._config.max_concurrency
        with self._executor_cls(max_workers=max_workers) as executor:
            upload_partial = functools.partial(
                self._upload_one_part, filename, bucket, key, upload_id,
                part_size, callback)
            for part in executor.map(upload_partial, range(1, num_parts + 1)):
                parts.append(part)
        # Parts have to be ordered by part number.
        parts.sort(key=lambda x: x['PartNumber'])
        self._client.complete_multipart_upload(
            Bucket=bucket, Key=key, UploadId=upload_id,
            MultipartUpload={'Parts': parts})

    def _upload_one_part(self, filename, bucket, key,
                         upload_id, part_size, callback, part_number):
        open_chunk_reader = self._os.open_file_chunk_reader
        with open_chunk_reader(filename, part_size * (part_number - 1),
                               part_size, callback) as body:
            response = self._client.upload_part(
                Bucket=bucket, Key=key,
                UploadId=upload_id, PartNumber=part_number, Body=body)
            etag = response['ETag']
            return {'ETag': etag, 'PartNumber': part_number}

class MultipartDownloader(object):
    def __init__(self, client, config, osutil,
                 executor_cls=futures.ThreadPoolExecutor):
        self._client = client
        self._config = config
        self._os = osutil
        self._executor_cls = executor_cls

    def download_file(self, bucket, key, filename, object_size,
                      callback=None):
        part_size = self._config.multipart_chunksize
        num_parts = int(math.ceil(object_size / float(part_size)))
        max_workers = self._config.max_concurrency
        with self._os.open(filename, 'wb') as f:
            thread_safe_fileobj = self._os.wrap_thread_safe_writer(f)
            download_partial = functools.partial(
                self._download_range, bucket, key, filename,
                part_size, num_parts, callback, thread_safe_fileobj)
            with self._executor_cls(max_workers=max_workers) as executor:
                list(executor.map(download_partial, range(num_parts)))

    def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, fileobj, i):
        start_range = i * part_size
        if i == num_parts - 1:
            end_range = ''
        else:
            end_range = start_range + part_size - 1
        range_param = 'bytes=%s-%s' % (start_range, end_range)
        response = self._client.get_object(
            Bucket=bucket, Key=key, Range=range_param)
        streaming_body = self._os.wrap_stream_with_callback(
            response['Body'], callback)
        buffer_size = 1024 * 16
        current_index = start_range
        for chunk in iter(lambda: streaming_body.read(buffer_size), b''):
            fileobj.pwrite(chunk, current_index)
            current_index += len(chunk)

class TransferConfig(object):
    def __init__(self,
                 multipart_threshold=8 * MB,
                 max_concurrency=2,
                 multipart_chunksize=8 * MB,
                 num_download_attempts=5):
        self.multipart_threshold = multipart_threshold
        self.max_concurrency = max_concurrency
        self.multipart_chunksize = multipart_chunksize
        self.num_download_attempts = num_download_attempts

class S3Transfer(object):

    def __init__(self, client, config=None, osutil=None):
        self._client = client
        if config is None:
            config = TransferConfig()
        self._config = config
        if osutil is None:
            osutil = OSUtils()
        self._osutil = osutil

    def upload_file(self, filename, bucket, key,
                    callback=None, extra_args=None):
        if extra_args is None:
            extra_args = {}
        if self._osutil.get_file_size(filename) >= \
                self._config.multipart_threshold:
            self._multipart_upload(filename, bucket, key, callback, extra_args)
        else:
            self._put_object(filename, bucket, key, callback, extra_args)

    def _put_object(self, filename, bucket, key, callback, extra_args):
        # We're using open_file_chunk_reader so we can take advantage of the
        # progress callback functionality.
        open_chunk_reader = self._osutil.open_file_chunk_reader
        with open_chunk_reader(filename, 0,
                               self._osutil.get_file_size(filename),
                               callback=callback) as body:
            self._client.put_object(Bucket=bucket, Key=key, Body=body,
                                    **extra_args)

    def download_file(self, bucket, key, filename, callback=None):
        """Download an S3 object to a file.

        This method will issue a ``head_object`` request to determine
        the size of the S3 object.  This is used to determine if the
        object is downloaded in parallel.

        """
        object_size = self._object_size(bucket, key)
        if object_size >= self._config.multipart_threshold:
            self._ranged_download(bucket, key, filename, object_size, callback)
        else:
            self._get_object(bucket, key, filename, callback)

    def _ranged_download(self, bucket, key, filename, object_size, callback):
        downloader = MultipartDownloader(self._client, self._config,
                                         self._osutil)
        downloader.download_file(bucket, key, filename, object_size,
                                 callback)

    def _get_object(self, bucket, key, filename, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                continue
        raise RetriesExceededError(e)

    def _do_get_object(self, bucket, key, filename, callback):
        response = self._client.get_object(Bucket=bucket, Key=key)
        streaming_body = self._osutil.wrap_stream_with_callback(
            response['Body'], callback)
        with self._osutil.open(filename, 'wb') as f:
            for chunk in iter(lambda: streaming_body.read(8192), b''):
                f.write(chunk)

    def _object_size(self, bucket, key):
        return self._client.head_object(
            Bucket=bucket, Key=key)['ContentLength']

    def _multipart_upload(self, filename, bucket, key, callback, extra_args):
        uploader = MultipartUploader(self._client, self._config, self._osutil)
        uploader.upload_file(filename, bucket, key, callback, extra_args)
<commit_msg>Update docs to reflect latest changes<commit_after>"""Abstractions over S3's upload/download operations.

This module provides high level abstractions for efficient
uploads/downloads.  It handles several things for the user:

* Automatically switching to multipart transfers when
  a file is over a specific size threshold
* Uploading/downloading a file in parallel
* Throttling based on max bandwidth
* Progress callbacks to monitor transfers
* Retries.  While botocore handles retries for streaming uploads,
  it is not possible for it to handle retries for streaming
  downloads.  This module handles retries for both cases so
  you don't need to implement any retry logic yourself.

This module has a reasonable set of defaults.  It also allows you
to configure many aspects of the transfer process including:

* Multipart threshold size
* Max parallel downloads
* Max bandwidth
* Socket timeouts
* Retry amounts

There is no support for s3->s3 multipart copies at this
time.

Usage
=====

The simplest way to use this module is:

.. code-block:: python

    client = boto3.client('s3', 'us-west-2')
    transfer = S3Transfer(client)
    # Upload /tmp/myfile to s3://bucket/key
    transfer.upload_file('/tmp/myfile', 'bucket', 'key')

    # Download s3://bucket/key to /tmp/myfile
    transfer.download_file('bucket', 'key', '/tmp/myfile')

The ``upload_file`` and ``download_file`` methods also accept
``**kwargs``, which will be forwarded through to the corresponding
client operation.  Here are a few examples using ``upload_file``::

    # Making the object public
    transfer.upload_file('/tmp/myfile', 'bucket', 'key',
                         extra_args={'ACL': 'public-read'})

    # Setting metadata
    transfer.upload_file('/tmp/myfile', 'bucket', 'key',
                         extra_args={'Metadata': {'a': 'b', 'c': 'd'}})

    # Setting content type
    transfer.upload_file('/tmp/myfile.json', 'bucket', 'key',
                         extra_args={'ContentType': "application/json"})

The ``S3Transfer`` clas also supports progress callbacks so you can
provide transfer progress to users.  Both the ``upload_file`` and
``download_file`` methods take an optional ``callback`` parameter.
Here's an example of how to print a simple progress percentage
to the user:

.. code-block:: python

    class ProgressPercentage(object):
        def __init__(self, filename):
            self._filename = filename
            self._size = float(os.path.getsize(filename))
            self._seen_so_far = 0
            self._lock = threading.Lock()

        def __call__(self, filename, bytes_amount):
            # To simplify we'll assume this is hooked up
            # to a single filename.
            with self._lock:
                self._seen_so_far += bytes_amount
                percentage = (self._seen_so_far / self._size) * 100
                sys.stdout.write(
                    "\r%s  %s / %s  (%.2f%%)" % (filename, self._seen_so_far,
                                                 self._size, percentage))
                sys.stdout.flush()

    transfer = S3Transfer(boto3.client('s3', 'us-west-2'))
    # Upload /tmp/myfile to s3://bucket/key and print upload progress.
    transfer.upload_file('/tmp/myfile', 'bucket', 'key',
                         callback=ProgressPercentage('/tmp/myfile'))

You can also provide an TransferConfig object to the S3Transfer
object that gives you more fine grained control over the
transfer.  For example:

.. code-block:: python

    client = boto3.client('s3', 'us-west-2')
    config = TransferConfig(
        multipart_threshold=8 * 1024 * 1024,
        max_concurrency=10,
        num_download_attempts=10,
    )
    transfer = S3Transfer(client, config)
    transfer.upload_file('/tmp/foo', 'bucket', 'key')

"""
import os
import math
import threading
import functools
import logging
import socket
from concurrent import futures

from botocore.vendored.requests.packages.urllib3.exceptions import \
    ReadTimeoutError
from botocore.exceptions import IncompleteReadError

from boto3.exceptions import RetriesExceededError

logger = logging.getLogger(__name__)
MB = 1024 * 1024

class ReadFileChunk(object):
    def __init__(self, fileobj, start_byte, chunk_size, full_file_size,
                 callback=None):
        """

        Given a file object shown below:

            |___________________________________________________|
            0          |                 |                 full_file_size
                       |----chunk_size---|
                 start_byte

        :type fileobj: file
        :param fileobj: File like object

        :type start_byte: int
        :param start_byte: The first byte from which to start reading.

        :type chunk_size: int
        :param chunk_size: The max chunk size to read.  Trying to read
            pass the end of the chunk size will behave like you've
            reached the end of the file.

        :type full_file_size: int
        :param full_file_size: The entire content length associated
            with ``fileobj``.

        :type callback: function(amount_read)
        :param callback: Called whenever data is read from this object.

        """
        self._fileobj = fileobj
        self._start_byte = start_byte
        self._size = self._calculate_file_size(
            self._fileobj, requested_size=chunk_size,
            start_byte=start_byte, actual_file_size=full_file_size)
        self._fileobj.seek(self._start_byte)
        self._amount_read = 0
        self._callback = callback

    @classmethod
    def from_filename(cls, filename, start_byte, chunk_size, callback=None):
        """Convenience factory function to create from a filename."""
        f = open(filename, 'rb')
        file_size = os.fstat(f.fileno()).st_size
        return cls(f, start_byte, chunk_size, file_size, callback)

    def _calculate_file_size(self, fileobj, requested_size, start_byte,
                             actual_file_size):
        max_chunk_size = actual_file_size - start_byte
        return min(max_chunk_size, requested_size)

    def read(self, amount=None):
        if amount is None:
            amount_to_read = self._size - self._amount_read
        else:
            amount_to_read = min(self._size - self._amount_read, amount)
        data = self._fileobj.read(amount_to_read)
        self._amount_read += len(data)
        if self._callback is not None:
            self._callback(len(data))
        return data

    def seek(self, where):
        self._fileobj.seek(self._start_byte + where)
        self._amount_read = where

    def close(self):
        self._fileobj.close()

    def tell(self):
        return self._amount_read

    def __len__(self):
        # __len__ is defined because requests will try to determine the length
        # of the stream to set a content length.  In the normal case
        # of the file it will just stat the file, but we need to change that
        # behavior.  By providing a __len__, requests will use that instead
        # of stat'ing the file.
        return self._size

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def __iter__(self):
        # This is a workaround for http://bugs.python.org/issue17575
        # Basically httplib will try to iterate over the contents, even
        # if its a file like object.  This wasn't noticed because we've
        # already exhausted the stream so iterating over the file immediately
        # steps, which is what we're simulating here.
        return iter([])

class StreamReaderProgress(object):
    """Wrapper for a read only stream that adds progress callbacks."""
    def __init__(self, stream, callback=None):
        self._stream = stream
        self._callback = callback

    def read(self, *args, **kwargs):
        value = self._stream.read(*args, **kwargs)
        if self._callback is not None:
            self._callback(len(value))
        return value

class ThreadSafeWriter(object):
    def __init__(self, write_stream):
        self._write_stream = write_stream
        self._lock = threading.Lock()

    def pwrite(self, data, offset):
        with self._lock:
            self._write_stream.seek(offset)
            self._write_stream.write(data)

    def close(self):
        return self._write_stream.close()

class OSUtils(object):
    def get_file_size(self, filename):
        return os.path.getsize(filename)

    def open_file_chunk_reader(self, filename, start_byte, size, callback):
        return ReadFileChunk.from_filename(filename, start_byte,
                                           size, callback)

    def open(self, filename, mode):
        return open(filename, mode)

    def wrap_stream_with_callback(self, stream, callback):
        return StreamReaderProgress(stream, callback)

    def wrap_thread_safe_writer(self, stream):
        return ThreadSafeWriter(stream)

class MultipartUploader(object):
    def __init__(self, client, config, osutil,
                 executor_cls=futures.ThreadPoolExecutor):
        self._client = client
        self._config = config
        self._os = osutil
        self._executor_cls = executor_cls

    def upload_file(self, filename, bucket, key, callback, extra_args):
        response = self._client.create_multipart_upload(Bucket=bucket,
                                                        Key=key, **extra_args)
        upload_id = response['UploadId']
        parts = []
        part_size = self._config.multipart_chunksize
        num_parts = int(
            math.ceil(self._os.get_file_size(filename) / float(part_size)))
        max_workers = self._config.max_concurrency
        with self._executor_cls(max_workers=max_workers) as executor:
            upload_partial = functools.partial(
                self._upload_one_part, filename, bucket, key, upload_id,
                part_size, callback)
            for part in executor.map(upload_partial, range(1, num_parts + 1)):
                parts.append(part)
        # Parts have to be ordered by part number.
        parts.sort(key=lambda x: x['PartNumber'])
        self._client.complete_multipart_upload(
            Bucket=bucket, Key=key, UploadId=upload_id,
            MultipartUpload={'Parts': parts})

    def _upload_one_part(self, filename, bucket, key,
                         upload_id, part_size, callback, part_number):
        open_chunk_reader = self._os.open_file_chunk_reader
        with open_chunk_reader(filename, part_size * (part_number - 1),
                               part_size, callback) as body:
            response = self._client.upload_part(
                Bucket=bucket, Key=key,
                UploadId=upload_id, PartNumber=part_number, Body=body)
            etag = response['ETag']
            return {'ETag': etag, 'PartNumber': part_number}

class MultipartDownloader(object):
    def __init__(self, client, config, osutil,
                 executor_cls=futures.ThreadPoolExecutor):
        self._client = client
        self._config = config
        self._os = osutil
        self._executor_cls = executor_cls

    def download_file(self, bucket, key, filename, object_size,
                      callback=None):
        part_size = self._config.multipart_chunksize
        num_parts = int(math.ceil(object_size / float(part_size)))
        max_workers = self._config.max_concurrency
        with self._os.open(filename, 'wb') as f:
            thread_safe_fileobj = self._os.wrap_thread_safe_writer(f)
            download_partial = functools.partial(
                self._download_range, bucket, key, filename,
                part_size, num_parts, callback, thread_safe_fileobj)
            with self._executor_cls(max_workers=max_workers) as executor:
                list(executor.map(download_partial, range(num_parts)))

    def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, fileobj, i):
        start_range = i * part_size
        if i == num_parts - 1:
            end_range = ''
        else:
            end_range = start_range + part_size - 1
        range_param = 'bytes=%s-%s' % (start_range, end_range)
        response = self._client.get_object(
            Bucket=bucket, Key=key, Range=range_param)
        streaming_body = self._os.wrap_stream_with_callback(
            response['Body'], callback)
        buffer_size = 1024 * 16
        current_index = start_range
        for chunk in iter(lambda: streaming_body.read(buffer_size), b''):
            fileobj.pwrite(chunk, current_index)
            current_index += len(chunk)

class TransferConfig(object):
    def __init__(self,
                 multipart_threshold=8 * MB,
                 max_concurrency=2,
                 multipart_chunksize=8 * MB,
                 num_download_attempts=5):
        self.multipart_threshold = multipart_threshold
        self.max_concurrency = max_concurrency
        self.multipart_chunksize = multipart_chunksize
        self.num_download_attempts = num_download_attempts

class S3Transfer(object):

    def __init__(self, client, config=None, osutil=None):
        self._client = client
        if config is None:
            config = TransferConfig()
        self._config = config
        if osutil is None:
            osutil = OSUtils()
        self._osutil = osutil

    def upload_file(self, filename, bucket, key,
                    callback=None, extra_args=None):
        if extra_args is None:
            extra_args = {}
        if self._osutil.get_file_size(filename) >= \
                self._config.multipart_threshold:
            self._multipart_upload(filename, bucket, key, callback, extra_args)
        else:
            self._put_object(filename, bucket, key, callback, extra_args)

    def _put_object(self, filename, bucket, key, callback, extra_args):
        # We're using open_file_chunk_reader so we can take advantage of the
        # progress callback functionality.
        open_chunk_reader = self._osutil.open_file_chunk_reader
        with open_chunk_reader(filename, 0,
                               self._osutil.get_file_size(filename),
                               callback=callback) as body:
            self._client.put_object(Bucket=bucket, Key=key, Body=body,
                                    **extra_args)

    def download_file(self, bucket, key, filename, callback=None):
        """Download an S3 object to a file.

        This method will issue a ``head_object`` request to determine
        the size of the S3 object.  This is used to determine if the
        object is downloaded in parallel.

        """
        object_size = self._object_size(bucket, key)
        if object_size >= self._config.multipart_threshold:
            self._ranged_download(bucket, key, filename, object_size, callback)
        else:
            self._get_object(bucket, key, filename, callback)

    def _ranged_download(self, bucket, key, filename, object_size, callback):
        downloader = MultipartDownloader(self._client, self._config,
                                         self._osutil)
        downloader.download_file(bucket, key, filename, object_size,
                                 callback)

    def _get_object(self, bucket, key, filename, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                continue
        raise RetriesExceededError(e)

    def _do_get_object(self, bucket, key, filename, callback):
        response = self._client.get_object(Bucket=bucket, Key=key)
        streaming_body = self._osutil.wrap_stream_with_callback(
            response['Body'], callback)
        with self._osutil.open(filename, 'wb') as f:
            for chunk in iter(lambda: streaming_body.read(8192), b''):
                f.write(chunk)

    def _object_size(self, bucket, key):
        return self._client.head_object(
            Bucket=bucket, Key=key)['ContentLength']

    def _multipart_upload(self, filename, bucket, key, callback, extra_args):
        uploader = MultipartUploader(self._client, self._config, self._osutil)
        uploader.upload_file(filename, bucket, key, callback, extra_args)
<|endoftext|>


<commit_before>from botstory import ast, matchers
from botstory.ast import forking
import logging
import json

logger = logging.getLogger(__name__)

# 1. once we trap on StoriesLoopNode we should stop execution
# and wait any user import
# 2. once we got any user import we should come to StoriesLoopNode
# add check whether input match received request.
# 3. execute matched story

class StoryLoopAPI:
    """
    loop (scope) concept similar to switch (forking)
    the main difference is that after case switch jump to the next part
    after last case. But in scope it loop until
    - we get receive unmatched message
    - or break loop explicitly
    """

    def __init__(self, library, parser_instance):
        self.library = library
        self.parser_instance = parser_instance

    def loop(self):
        def fn(one_loop):
            scope_node = StoriesLoopNode(one_loop)
            self.parser_instance.add_to_current_node(scope_node)
            self.parser_instance.compile_scope(scope_node, one_loop)
            # TODO: crawl scope for matchers and handlers

            # 1) we already have hierarchy of stories and stack of execution
            # it works that way -- we trying to match request to some story validator
            # by getting story one-by-one from stack

            # 2) we cold add validator for catching all stories from one scope

            # 3) if we didn't match scope we bubble up to previous scope

            # 4) if we match scope-validator we should choose one of its story

            return one_loop

        return fn

class StoriesLoopNode:
    def __init__(self, target):
        self.target = target
        self.local_scope = ast.library.StoriesScope()

    @property
    def __name__(self):
        return self.target.__name__

    @property
    def children(self):
        return self.local_scope.stories

    def should_loop(self):
        return True

    def __call__(self, *args, **kwargs):
        return ScopeMatcher(forking.Switch(self.local_scope.all_filters()))

    def get_child_by_validation_result(self, topic):
        case_stories = self.local_scope.by_topic(topic)

        if len(case_stories) == 0:
            logger.debug('#######################################')
            logger.debug('# [!] do not have any child here')
            logger.debug('# story node = {}'.format(self))
            logger.debug('# topic = {}'.format(topic))
            logger.debug('#######################################')
            return None

        return case_stories[0]

    def match_children(self, key, value):
        return [child for child in self.local_scope.stories
                if child.extensions.get(key, forking.Undefined) == value]

    def to_json(self):
        return {
            'type': 'StoriesLoopNode',
            'name': self.__name__,
            'local_scope': self.local_scope.to_json()
        }

    def __repr__(self):
        return json.dumps(self.to_json())

@matchers.matcher()
class ScopeMatcher:
    def __init__(self, all_filters):
        # TODO: It's not good name
        # but better what we have for this moment
        # so it could be renamed very soon
        self.new_scope = True
        self.all_filters = all_filters

    def validate(self, message):
        return self.all_filters.validate(message)

    def serialize(self):
        return self.all_filters.serialize()

    @classmethod
    def deserialize(cls, data):
        return cls(forking.Switch.deserialize(data))
<commit_msg>remove extra comments<commit_after>from botstory import ast, matchers
from botstory.ast import forking
import logging
import json

logger = logging.getLogger(__name__)

# 1. once we trap on StoriesLoopNode we should stop execution
# and wait any user input
# 2. once we got any user import we should come to StoriesLoopNode
# add check whether input match received request.
# 3. execute matched story

class StoryLoopAPI:
    """
    loop (scope) concept similar to switch (forking)
    the main difference is that after case switch jump to the next part
    after last case. But in scope it loop until
    - we get receive unmatched message
    - or break loop explicitly
    """

    def __init__(self, library, parser_instance):
        self.library = library
        self.parser_instance = parser_instance

    def loop(self):
        def fn(one_loop):
            scope_node = StoriesLoopNode(one_loop)
            self.parser_instance.add_to_current_node(scope_node)
            self.parser_instance.compile_scope(scope_node, one_loop)
            return one_loop

        return fn

class StoriesLoopNode:
    def __init__(self, target):
        self.target = target
        self.local_scope = ast.library.StoriesScope()

    @property
    def __name__(self):
        return self.target.__name__

    @property
    def children(self):
        return self.local_scope.stories

    def should_loop(self):
        return True

    def __call__(self, *args, **kwargs):
        return ScopeMatcher(forking.Switch(self.local_scope.all_filters()))

    def get_child_by_validation_result(self, topic):
        case_stories = self.local_scope.by_topic(topic)

        if len(case_stories) == 0:
            logger.debug('#######################################')
            logger.debug('# [!] do not have any child here')
            logger.debug('# story node = {}'.format(self))
            logger.debug('# topic = {}'.format(topic))
            logger.debug('#######################################')
            return None

        return case_stories[0]

    def match_children(self, key, value):
        return [child for child in self.local_scope.stories
                if child.extensions.get(key, forking.Undefined) == value]

    def to_json(self):
        return {
            'type': 'StoriesLoopNode',
            'name': self.__name__,
            'local_scope': self.local_scope.to_json()
        }

    def __repr__(self):
        return json.dumps(self.to_json())

@matchers.matcher()
class ScopeMatcher:
    def __init__(self, all_filters):
        # TODO: It's not good name
        # but better what we have for this moment
        # so it could be renamed very soon
        self.new_scope = True
        self.all_filters = all_filters

    def validate(self, message):
        return self.all_filters.validate(message)

    def serialize(self):
        return self.all_filters.serialize()

    @classmethod
    def deserialize(cls, data):
        return cls(forking.Switch.deserialize(data))
<|endoftext|>


<commit_before>"""Metadata about bouncer shared between setup.py and bouncer code."""

__all__ = ["__version__"]

__version__ = "0.0.4"  # PEP440-compliant version number.
<commit_msg>Bump version to 0.0.5<commit_after>"""Metadata about bouncer shared between setup.py and bouncer code."""

__all__ = ["__version__"]

__version__ = "0.0.5"  # PEP440-compliant version number.
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from empiricaldist import Pmf

def values(series):
    """Make a series of values and the number of times they appear.

    Returns a DataFrame because they get rendered better in Jupyter.

    series: Pandas Series

    returns: Pandas DataFrame
    """
    series = series.value_counts(dropna=False).sort_index()
    series.index.name = 'values'
    series.name = 'counts'
    return pd.DataFrame(series)

def write_table(table, label, **options):
    """Write a table in LaTex format.

    table: DataFrame
    label: string
    options: passed to DataFrame.to_latex
    """
    filename = f'tables/{label}.tex'
    fp = open(filename, 'w')
    s = table.to_latex(**options)
    fp.write(s)
    fp.close()

def write_pmf(pmf, label):
    """Write a Pmf object as a table.

    pmf: Pmf
    label: string
    """
    df = pd.DataFrame()
    df['qs'] = pmf.index
    df['ps'] = pmf.values
    write_table(df, label, index=False)

def underride(d, **options):
    """Add key-value pairs to d only if key is not in d.

    d: dictionary
    options: keyword args to add to d
    """
    for key, val in options.items():
        d.setdefault(key, val)

    return d

def decorate(**options):
    """Decorate the current axes.

    Call decorate with keyword arguments like
    decorate(title='Title',
             xlabel='x',
             ylabel='y')

    The keyword arguments can be any of the axis properties
    https://matplotlib.org/api/axes_api.html
    """
    ax = plt.gca()
    ax.set(**options)

    handles, labels = ax.get_legend_handles_labels()
    if handles:
        ax.legend(handles, labels)

    plt.tight_layout()

def savefig(root, **options):
    """Save the current figure.

    root: string filename root
    options: passed to plt.savefig
    """
    format = options.pop('format', None)
    if format:
        formats = [format]
    else:
        formats = ['pdf', 'png']

    for format in formats:
        fname = f'figs/{root}.{format}'
        plt.savefig(fname, **options)

def make_mixture(pmf, pmf_seq):
    """Make a mixture of distributions.

    pmf: mapping from each hypothesis to its probability
    pmf_seq: sequence of Pmfs, each representing
             a conditional distribution for one hypothesis

    returns: Pmf representing the mixture
    """
    df = pd.DataFrame(pmf_seq).fillna(0).transpose()
    df *= pmf.ps
    total = df.sum(axis=1)
    return Pmf(total)

def outer_product(s1, s2):
    """Compute the outer product of two Series.

    First Series goes down the rows;
    second goes across the columns.

    s1: Series
    s2: Series

    return: DataFrame
    """
    a = np.multiply.outer(s1.to_numpy(), s2.to_numpy())
    return pd.DataFrame(a, index=s1.index, columns=s2.index)

def make_uniform(start, stop, num=51, name=None, **options):
    """Make a Pmf that represents a uniform distribution.

    start: lower bound
    stop: upper bound
    num: number of points
    name: string name for the quantities
    options: passed to Pmf

    returns: Pmf
    """
    qs = np.linspace(start, stop, num)
    pmf = Pmf(1.0, qs, **options)
    pmf.normalize()
    if name:
        pmf.index.name = name
    return pmf

def make_joint(s1, s2):
    """Compute the outer product of two Series.

    First Series goes across the columns;
    second goes down the rows.

    s1: Series
    s2: Series

    return: DataFrame
    """
    X, Y = np.meshgrid(s1, s2)
    return pd.DataFrame(X*Y, columns=s1.index, index=s2.index)

def make_mesh(joint):
    """Make a mesh grid from the quantities in a joint distribution.

    joint: DataFrame representing a joint distribution

    returns: a mesh grid (X, Y) where X contains the column names and
                                      Y contains the row labels
    """
    x = joint.columns
    y = joint.index
    return np.meshgrid(x, y)

def normalize(joint):
    """Normalize a joint distribution.

    joint: DataFrame
    """
    prob_data = joint.to_numpy().sum()
    joint /= prob_data
    return prob_data

def marginal(joint, axis):
    """Compute a marginal distribution.

    axis=0 returns the marginal distribution of the first variable
    axis=1 returns the marginal distribution of the second variable

    joint: DataFrame representing a joint distribution
    axis: int axis to sum along

    returns: Pmf
    """
    return Pmf(joint.sum(axis=axis))

def pmf_marginal(joint_pmf, level):
    """Compute a marginal distribution.

    joint_pmf: Pmf representing a joint distribution
    level: int, level to sum along

    returns: Pmf
    """
    return Pmf(joint_pmf.sum(level=level))

def plot_contour(joint, **options):
    """Plot a joint distribution.

    joint: DataFrame representing a joint PMF
    """
    cs = plt.contour(joint.columns, joint.index, joint, **options)
    decorate(xlabel=joint.columns.name,
             ylabel=joint.index.name)
    return cs

from scipy.stats import binom

def make_binomial(n, p):
    """Make a binomial distribution.

    n: number of trials
    p: probability of success

    returns: Pmf representing the distribution of k
    """
    ks = np.arange(n+1)
    ps = binom.pmf(ks, n, p)
    return Pmf(ps, ks)

from statsmodels.nonparametric.smoothers_lowess import lowess

def make_lowess(series):
    """Use LOWESS to compute a smooth line.

    series: pd.Series

    returns: pd.Series
    """
    endog = series.values
    exog = series.index.values

    smooth = lowess(endog, exog)
    index, data = np.transpose(smooth)

    return pd.Series(data, index=index)

def plot_series_lowess(series, color):
    """Plots a series of data points and a smooth line.

    series: pd.Series
    color: string or tuple
    """
    series.plot(lw=0, marker='o', color=color, alpha=0.5)
    smooth = make_lowess(series)
    smooth.plot(label='_', color=color)
<commit_msg>Adding joint_grid<commit_after>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from empiricaldist import Pmf

def values(series):
    """Make a series of values and the number of times they appear.

    Returns a DataFrame because they get rendered better in Jupyter.

    series: Pandas Series

    returns: Pandas DataFrame
    """
    series = series.value_counts(dropna=False).sort_index()
    series.index.name = 'values'
    series.name = 'counts'
    return pd.DataFrame(series)

def write_table(table, label, **options):
    """Write a table in LaTex format.

    table: DataFrame
    label: string
    options: passed to DataFrame.to_latex
    """
    filename = f'tables/{label}.tex'
    fp = open(filename, 'w')
    s = table.to_latex(**options)
    fp.write(s)
    fp.close()

def write_pmf(pmf, label):
    """Write a Pmf object as a table.

    pmf: Pmf
    label: string
    """
    df = pd.DataFrame()
    df['qs'] = pmf.index
    df['ps'] = pmf.values
    write_table(df, label, index=False)

def underride(d, **options):
    """Add key-value pairs to d only if key is not in d.

    d: dictionary
    options: keyword args to add to d
    """
    for key, val in options.items():
        d.setdefault(key, val)

    return d

def decorate(**options):
    """Decorate the current axes.

    Call decorate with keyword arguments like
    decorate(title='Title',
             xlabel='x',
             ylabel='y')

    The keyword arguments can be any of the axis properties
    https://matplotlib.org/api/axes_api.html
    """
    ax = plt.gca()
    ax.set(**options)

    handles, labels = ax.get_legend_handles_labels()
    if handles:
        ax.legend(handles, labels)

    plt.tight_layout()

def savefig(root, **options):
    """Save the current figure.

    root: string filename root
    options: passed to plt.savefig
    """
    format = options.pop('format', None)
    if format:
        formats = [format]
    else:
        formats = ['pdf', 'png']

    for format in formats:
        fname = f'figs/{root}.{format}'
        plt.savefig(fname, **options)

def make_mixture(pmf, pmf_seq):
    """Make a mixture of distributions.

    pmf: mapping from each hypothesis to its probability
    pmf_seq: sequence of Pmfs, each representing
             a conditional distribution for one hypothesis

    returns: Pmf representing the mixture
    """
    df = pd.DataFrame(pmf_seq).fillna(0).transpose()
    df *= pmf.ps
    total = df.sum(axis=1)
    return Pmf(total)

def outer_product(s1, s2):
    """Compute the outer product of two Series.

    First Series goes down the rows;
    second goes across the columns.

    s1: Series
    s2: Series

    return: DataFrame
    """
    a = np.multiply.outer(s1.to_numpy(), s2.to_numpy())
    return pd.DataFrame(a, index=s1.index, columns=s2.index)

def make_uniform(start, stop, num=51, name=None, **options):
    """Make a Pmf that represents a uniform distribution.

    start: lower bound
    stop: upper bound
    num: number of points
    name: string name for the quantities
    options: passed to Pmf

    returns: Pmf
    """
    qs = np.linspace(start, stop, num)
    pmf = Pmf(1.0, qs, **options)
    pmf.normalize()
    if name:
        pmf.index.name = name
    return pmf

def make_joint(s1, s2):
    """Compute the outer product of two Series.

    First Series goes across the columns;
    second goes down the rows.

    s1: Series
    s2: Series

    return: DataFrame
    """
    X, Y = np.meshgrid(s1, s2)
    return pd.DataFrame(X*Y, columns=s1.index, index=s2.index)

def make_mesh(joint):
    """Make a mesh grid from the quantities in a joint distribution.

    joint: DataFrame representing a joint distribution

    returns: a mesh grid (X, Y) where X contains the column names and
                                      Y contains the row labels
    """
    x = joint.columns
    y = joint.index
    return np.meshgrid(x, y)

def normalize(joint):
    """Normalize a joint distribution.

    joint: DataFrame
    """
    prob_data = joint.to_numpy().sum()
    joint /= prob_data
    return prob_data

def marginal(joint, axis):
    """Compute a marginal distribution.

    axis=0 returns the marginal distribution of the first variable
    axis=1 returns the marginal distribution of the second variable

    joint: DataFrame representing a joint distribution
    axis: int axis to sum along

    returns: Pmf
    """
    return Pmf(joint.sum(axis=axis))

def pmf_marginal(joint_pmf, level):
    """Compute a marginal distribution.

    joint_pmf: Pmf representing a joint distribution
    level: int, level to sum along

    returns: Pmf
    """
    return Pmf(joint_pmf.sum(level=level))

def plot_contour(joint, **options):
    """Plot a joint distribution.

    joint: DataFrame representing a joint PMF
    """
    cs = plt.contour(joint.columns, joint.index, joint, **options)
    decorate(xlabel=joint.columns.name,
             ylabel=joint.index.name)
    return cs

from scipy.stats import binom

def make_binomial(n, p):
    """Make a binomial distribution.

    n: number of trials
    p: probability of success

    returns: Pmf representing the distribution of k
    """
    ks = np.arange(n+1)
    ps = binom.pmf(ks, n, p)
    return Pmf(ps, ks)

from statsmodels.nonparametric.smoothers_lowess import lowess

def make_lowess(series):
    """Use LOWESS to compute a smooth line.

    series: pd.Series

    returns: pd.Series
    """
    endog = series.values
    exog = series.index.values

    smooth = lowess(endog, exog)
    index, data = np.transpose(smooth)

    return pd.Series(data, index=index)

def plot_series_lowess(series, color):
    """Plots a series of data points and a smooth line.

    series: pd.Series
    color: string or tuple
    """
    series.plot(lw=0, marker='o', color=color, alpha=0.5)
    smooth = make_lowess(series)
    smooth.plot(label='_', color=color)

from seaborn import JointGrid

def joint_plot(joint, **options):
    """Show joint and marginal distributions.

    joint: DataFrame that represents a joint distribution
    options: passed to JointGrid
    """
    # get the names of the parameters
    x = joint.columns.name
    x = 'x' if x is None else x

    y = joint.index.name
    y = 'y' if y is None else y

    # make a JointGrid with minimal data
    data = pd.DataFrame({x:[0], y:[0]})
    g = JointGrid(x, y, data, **options)

    # replace the contour plot
    g.ax_joint.contour(joint.columns,
                       joint.index,
                       joint,
                       cmap='viridis')

    # replace the marginals
    marginal_x = marginal(joint, 0)
    g.ax_marg_x.plot(marginal_x.qs, marginal_x.ps)

    marginal_y = marginal(joint, 1)
    g.ax_marg_y.plot(marginal_y.ps, marginal_y.qs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Bowtie Component classes, all visual and control components inherit these
"""

from __future__ import unicode_literals

from builtins import bytes

import json
import msgpack
from datetime import datetime, date, time

import flask
from flask_socketio import emit
from future.utils import with_metaclass
from eventlet.event import Event
from eventlet.queue import LightQueue

from bowtie._compat import IS_PY35

def json_conversion(obj):

    if isinstance(obj, datetime) or isinstance(obj, time) or isinstance(obj, date):
        return obj.isoformat()
    raise TypeError('Not sure how to serialize {} of type {}'.format(obj, type(obj)))

def jdumps(data):
    return json.dumps(data, default=json_conversion)

def encoders(obj):
    if isinstance(obj, datetime) or isinstance(obj, time) or isinstance(obj, date):
        return obj.isoformat()
    return obj

def pack(x):
    return bytes(msgpack.packb(x, default=encoders))

def make_event(event):

    @property
    def actualevent(self):
        name = event.__name__[3:]
        return '{uuid}#{event}'.format(uuid=self._uuid, event=name)

    if IS_PY35:
        # can't set docstring on properties in python 2 like this
        actualevent.__doc__ = event.__doc__

    return actualevent

def is_event(attribute):
    return attribute.startswith('on_')

def make_command(command):

    def actualcommand(self, data):
        name = command.__name__[3:]
        signal = '{uuid}#{event}'.format(uuid=self._uuid, event=name)
        if flask.has_request_context():
            # return emit(signal, jdumps(data))
            print(signal)
            return emit(signal, {'data': pack(data)})
        else:
            sio = flask.current_app.extensions['socketio']
            # return sio.emit(signal, jdumps(data))
            return sio.emit(signal, {'data': pack(data)})

    actualcommand.__doc__ = command.__doc__

    return actualcommand

def is_command(attribute):
    return attribute.startswith('do_')

class _Maker(type):
    def __new__(cls, name, parents, dct):
        for k in dct:
            if is_event(k):
                dct[k] = make_event(dct[k])
            if is_command(k):
                dct[k] = make_command(dct[k])
        return super(_Maker, cls).__new__(cls, name, parents, dct)

class Component(with_metaclass(_Maker, object)):
    _NEXT_UUID = 0

    @classmethod
    def _next_uuid(cls):
        cls._NEXT_UUID += 1
        return cls._NEXT_UUID

    def __init__(self):
        # wanted to put "self" instead of "Component"
        # was surprised that didn't work
        self._uuid = Component._next_uuid()
        super(Component, self).__init__()

    def get(self, block=True, timeout=None):
        event = LightQueue(1)
        if flask.has_request_context():
            emit('{}#get'.format(self._uuid),
                 callback=lambda x: event.put(msgpack.unpackb(bytes(x['data']))))
        else:
            sio = flask.current_app.extensions['socketio']
            sio.emit('{}#get'.format(self._uuid),
                     callback=lambda x: event.put(msgpack.unpackb(bytes(x['data']))))
        return event.get(timeout=10)
<commit_msg>serialize numpy ndarray<commit_after># -*- coding: utf-8 -*-
"""
Bowtie Component classes, all visual and control components inherit these
"""

from __future__ import unicode_literals

from builtins import bytes

import json
import msgpack
from datetime import datetime, date, time

import flask
from flask_socketio import emit
from future.utils import with_metaclass
from eventlet.event import Event
from eventlet.queue import LightQueue

from bowtie._compat import IS_PY35

def json_conversion(obj):
    try:
        # numpy isn't an explicit dependency of bowtie
        # so we can't assume it's available
        import numpy as np
        if isinstance(obj, np.ndarray):
            return obj.tolist()
    except ImportError:
        pass

    if isinstance(obj, datetime) or isinstance(obj, time) or isinstance(obj, date):
        return obj.isoformat()
    raise TypeError('Not sure how to serialize {} of type {}'.format(obj, type(obj)))

def jdumps(data):
    return json.dumps(data, default=json_conversion)

def encoders(obj):
    try:
        # numpy isn't an explicit dependency of bowtie
        # so we can't assume it's available
        import numpy as np
        if isinstance(obj, np.ndarray):
            return obj.tolist()
    except ImportError:
        pass

    if isinstance(obj, datetime) or isinstance(obj, time) or isinstance(obj, date):
        return obj.isoformat()

    return obj

def pack(x):
    return bytes(msgpack.packb(x, default=encoders))

def make_event(event):

    @property
    def actualevent(self):
        name = event.__name__[3:]
        return '{uuid}#{event}'.format(uuid=self._uuid, event=name)

    if IS_PY35:
        # can't set docstring on properties in python 2 like this
        actualevent.__doc__ = event.__doc__

    return actualevent

def is_event(attribute):
    return attribute.startswith('on_')

def make_command(command):

    def actualcommand(self, data):
        name = command.__name__[3:]
        signal = '{uuid}#{event}'.format(uuid=self._uuid, event=name)
        if flask.has_request_context():
            return emit(signal, {'data': pack(data)})
        else:
            sio = flask.current_app.extensions['socketio']
            return sio.emit(signal, {'data': pack(data)})

    actualcommand.__doc__ = command.__doc__

    return actualcommand

def is_command(attribute):
    return attribute.startswith('do_')

class _Maker(type):
    def __new__(cls, name, parents, dct):
        for k in dct:
            if is_event(k):
                dct[k] = make_event(dct[k])
            if is_command(k):
                dct[k] = make_command(dct[k])
        return super(_Maker, cls).__new__(cls, name, parents, dct)

class Component(with_metaclass(_Maker, object)):
    _NEXT_UUID = 0

    @classmethod
    def _next_uuid(cls):
        cls._NEXT_UUID += 1
        return cls._NEXT_UUID

    def __init__(self):
        # wanted to put "self" instead of "Component"
        # was surprised that didn't work
        self._uuid = Component._next_uuid()
        super(Component, self).__init__()

    def get(self, block=True, timeout=None):
        event = LightQueue(1)
        if flask.has_request_context():
            emit('{}#get'.format(self._uuid),
                 callback=lambda x: event.put(msgpack.unpackb(bytes(x['data']))))
        else:
            sio = flask.current_app.extensions['socketio']
            sio.emit('{}#get'.format(self._uuid),
                     callback=lambda x: event.put(msgpack.unpackb(bytes(x['data']))))
        return event.get(timeout=10)
<|endoftext|>


<commit_before>import webbrowser

import click
import pandas as pd
import json
import sys
from dateutil.parser import parse as dateutil_parse
from jinja2 import Template

pd.options.display.max_colwidth = 100

def parse_log(file_path):
    df = pd.read_csv(file_path, sep=",",
                     names=['Level', 'Timestamp', 'External ID', 'Prop', 'QID', 'Message', 'Msg Type'],
                     dtype={'External ID': str}, comment='#', quotechar='"', skipinitialspace=True, delimiter=';')
    df.fillna('', inplace=True)
    df = df.apply(lambda x: x.str.strip())
    df.Timestamp = pd.to_datetime(df.Timestamp, format='%m/%d/%Y %H:%M:%S')
    return df

def process_log(file_path):
    """
    Expects header as first line in log file. Header begins with comment character '#'. The line is a json string dump of
    a dictionary that contains the following keys:
    name: str, Task name
    maintainer: str, Name of person
    tags: list of tags associated with the task. can be empty
    properties: list of properties associated with the task. can be empty
    run_id: str, a run ID for the task run
    timestamp: str, timestamp for the task run

    :param file_path:
    :return:
    """
    # read header
    if isinstance(file_path, str):
        with open(file_path) as f:
            line = f.readline()
    else:
        line = file_path.readline()
    if not line.startswith("#"):
        raise ValueError("Expecting header in log file")
    metadata = json.loads(line[1:])
    if 'timestamp' in metadata:
        metadata['timestamp'] = dateutil_parse(metadata['timestamp'])

    df = parse_log(file_path)
    return df, metadata

def generate_summary(df):
    level_counts = df.Level.value_counts().to_dict()
    zlist = list(zip(*[('Items Processed Succesfully', level_counts.get('INFO', 0)),
                       ('Items Skipped Due to Missing Annotations ("Warning")', level_counts.get('WARNING', 0)),
                       ('Items Skipped Due to an Error', level_counts.get('ERROR', 0))]))
    level_counts = pd.Series(zlist[1], index=zlist[0])
    level_counts.name = "Count"

    info_counts = df.query("Level == 'INFO'").Message.value_counts().to_dict()
    zlist = list(zip(*[('No Action', info_counts.get('SKIP', 0)),
                       ('Updated', info_counts.get('UPDATE', 0)),
                       ('Created', info_counts.get('CREATE', 0))]))
    info_counts = pd.Series(zlist[1], index=zlist[0])
    info_counts.name = "Count"

    warning_counts = df.query("Level == 'WARNING'")['Msg Type'].value_counts()
    warning_counts.name = "Count"
    error_counts = df.query("Level == 'ERROR'")['Msg Type'].value_counts()
    error_counts.name = "Count"
    return level_counts, info_counts, warning_counts, error_counts

def isint(s):
    try:
        int(s)
    except Exception:
        return False
    return True

def url_qid(df, col):
    href = "{}"
    f = lambda x: href.format(x, x) if x.startswith("Q") and isint(x[1:]) else x
    df.is_copy = False
    df.loc[:, col] = df[col].apply(f)
    return df

@click.command()
@click.argument('log-path')
@click.option('--show-browser', default=False, is_flag=True, help='show log in browser')
def main(log_path, show_browser=False):
    print(log_path)
    df, metadata = process_log(log_path)
    level_counts, info_counts, warning_counts, error_counts = generate_summary(df)
    warnings_df = df.query("Level == 'WARNING'")
    errors_df = df.query("Level == 'ERROR'")
    info_df = df.query("Level == 'INFO'")
    info_df = url_qid(info_df, "QID")

    template = Template(open("template.html").read())
    s = template.render(name=metadata['name'], run_id=metadata['run_id'],
                        level_counts=level_counts.to_frame().to_html(),
                        info_counts=info_counts.to_frame().to_html(),
                        warning_counts=warning_counts.to_frame().to_html(),
                        error_counts=error_counts.to_frame().to_html(),
                        warnings_df=warnings_df.to_html(), errors_df=errors_df.to_html(),
                        info_df=info_df.to_html(escape=False))
    out_path = log_path.rsplit(".", 1)[0] + ".html"
    with open(out_path, 'w') as f:
        f.write(s)

    if show_browser:
        webbrowser.open(out_path)

if __name__ == "__main__":
    main()

<commit_msg>run all files in folder<commit_after>import glob
import os
import webbrowser

import click
import pandas as pd
import json
import sys
from dateutil.parser import parse as dateutil_parse
from jinja2 import Template

pd.options.display.max_colwidth = 100

def parse_log(file_path):
    df = pd.read_csv(file_path, sep=",",
                     names=['Level', 'Timestamp', 'External ID', 'Prop', 'QID', 'Message', 'Msg Type'],
                     dtype={'External ID': str}, comment='#', quotechar='"', skipinitialspace=True, delimiter=';')
    df.fillna('', inplace=True)
    df = df.apply(lambda x: x.str.strip())
    df.Timestamp = pd.to_datetime(df.Timestamp, format='%m/%d/%Y %H:%M:%S')
    return df

def process_log(file_path):
    """
    Expects header as first line in log file. Header begins with comment character '#'. The line is a json string dump of
    a dictionary that contains the following keys:
    name: str, Task name
    maintainer: str, Name of person
    tags: list of tags associated with the task. can be empty
    properties: list of properties associated with the task. can be empty
    run_id: str, a run ID for the task run
    timestamp: str, timestamp for the task run

    :param file_path:
    :return:
    """
    # read header
    if isinstance(file_path, str):
        with open(file_path) as f:
            line = f.readline()
    else:
        line = file_path.readline()
    if not line.startswith("#"):
        raise ValueError("Expecting header in log file")
    metadata = json.loads(line[1:])
    if 'timestamp' in metadata:
        metadata['timestamp'] = dateutil_parse(metadata['timestamp'])

    df = parse_log(file_path)
    return df, metadata

def generate_summary(df):
    level_counts = df.Level.value_counts().to_dict()
    zlist = list(zip(*[('Items Processed Succesfully', level_counts.get('INFO', 0)),
                       ('Items Skipped Due to Missing Annotations ("Warning")', level_counts.get('WARNING', 0)),
                       ('Items Skipped Due to an Error', level_counts.get('ERROR', 0))]))
    level_counts = pd.Series(zlist[1], index=zlist[0])
    level_counts.name = "Count"

    info_counts = df.query("Level == 'INFO'").Message.value_counts().to_dict()
    zlist = list(zip(*[('No Action', info_counts.get('SKIP', 0)),
                       ('Updated', info_counts.get('UPDATE', 0)),
                       ('Created', info_counts.get('CREATE', 0))]))
    info_counts = pd.Series(zlist[1], index=zlist[0])
    info_counts.name = "Count"

    warning_counts = df.query("Level == 'WARNING'")['Msg Type'].value_counts()
    warning_counts.name = "Count"
    error_counts = df.query("Level == 'ERROR'")['Msg Type'].value_counts()
    error_counts.name = "Count"
    return level_counts, info_counts, warning_counts, error_counts

def isint(s):
    try:
        int(s)
    except Exception:
        return False
    return True

def url_qid(df, col):
    href = "{}"
    f = lambda x: href.format(x, x) if x.startswith("Q") and isint(x[1:]) else x
    df.is_copy = False
    df.loc[:, col] = df[col].apply(f)
    return df

@click.command()
@click.argument('log-path')
@click.option('--show-browser', default=False, is_flag=True, help='show log in browser')
def main(log_path, show_browser=False):
    if os.path.isdir(log_path):
        # run on all files in dir (that don't end in html), and ignore show_browser
        files = [x for x in glob.glob(os.path.join(log_path, "*")) if not x.endswith(".html")]
        print(files)
        for file in files:
            _main(file)
    else:
        _main(log_path, show_browser)

def _main(log_path, show_browser=False):
    print(log_path)
    df, metadata = process_log(log_path)
    level_counts, info_counts, warning_counts, error_counts = generate_summary(df)
    warnings_df = df.query("Level == 'WARNING'")
    errors_df = df.query("Level == 'ERROR'")
    info_df = df.query("Level == 'INFO'")
    info_df = url_qid(info_df, "QID")

    template = Template(open(os.path.join(sys.path[0], "template.html")).read())
    s = template.render(name=metadata['name'], run_id=metadata['run_id'],
                        level_counts=level_counts.to_frame().to_html(),
                        info_counts=info_counts.to_frame().to_html(),
                        warning_counts=warning_counts.to_frame().to_html(),
                        error_counts=error_counts.to_frame().to_html(),
                        warnings_df=warnings_df.to_html(), errors_df=errors_df.to_html(),
                        info_df=info_df.to_html(escape=False))
    out_path = log_path.rsplit(".", 1)[0] + ".html"
    with open(out_path, 'w') as f:
        f.write(s)

    if show_browser:
        webbrowser.open(out_path)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>
inset('btcChain.py')

# btcrelay can relay a transaction to any contract that has a function
# name 'processTransaction' with signature si:i
extern relayDestination: [processTransaction:si:i]

# note: _ancestor[9]
#
# a Bitcoin block (header) is stored as:
# - _blockHeader 80 bytes
# - _height is 1 more than the typical Bitcoin term height/blocknumber [see setPreGensesis()]
# - _score is 1 more than the cumulative difficulty [see setPreGenesis()]
# - _ancestor list for more efficient backtracking (see btcChain)
data block[2^256](_height, _score, _ancestor[9], _blockHeader[])

# block with the highest score (aka the Head of the blockchain)
data heaviestBlock

# highest score among all blocks (so far)
data highScore

# owner/adminstrator of this contract
data owner

def init():
    self.owner = msg.sender
    # TODO anything else to init ?

#TODO for testing only; should be omitted for production
def testingonlySetHeaviest(blockHash):
    if tx.origin == self.owner:
        self.heaviestBlock = blockHash

# this has 2 purposes:
# 1) this should only be called once and is to "store" the block
# with hash zeroes, so that storing the real genesis
# block can be done using storeBlockHeader() instead of a special case
#
# 2) allows testing of storing arbitrary headers and verifying/relaying
# transactions, say from block 300K, instead
# of Satoshi's genesis block which have 0 transactions until much later on
#
# Since zero is the value that indicates non-existence, block _height
# and _score have to be set carefully and they have these consequences:
#
# - _height is 1 more than the typical Bitcoin
# term height/blocknumber. ie genesis has height 1 instead of 0
#
# - _score is 1 more than the cumulative "Bitcoin standard
# difficulty". eg the cumulative difficulty at the block after genesis
# is 2 (it and genesis have difficulty of 1), but this contract assigns
# a score of 3 for the block after genesis [see testStoringHeaders()]
def setPreGenesis(blockHash):
    if tx.origin == self.owner:
        self.heaviestBlock = blockHash

        # _height cannot be set to -1 because inMainChain() assumes that
        # a block with height0 does NOT exist, thus we cannot allow the
        # real genesis block to be at height0
        self.block[blockHash]._height = 0

        # set score to 1, since score0 means block does NOT exist. see check in storeBlockHeader()
        # this means that the score of a block is 1 more than the
        # cumulative difficulty to that block
        self.block[blockHash]._score = 1

        ancLen = self.numAncestorDepths
        i = 0
        while i < ancLen:
            self.block[blockHash]._ancestor[i] = blockHash
            i += 1

# store a Bitcoin block header that must be provided in
# binary format 'blockHeaderBinary'
def storeBlockHeader(blockHeaderBinary:str):
    hashPrevBlock = getBytesLE(blockHeaderBinary, 32, 4)

    assert self.block[hashPrevBlock]._score  # score0 means block does NOT exist; genesis has score of 1

    blockHash = self.fastHashBlock(blockHeaderBinary)

    if self.block[blockHash]._score != 0:  # block already exists
        return(0)

    bits = getBytesLE(blockHeaderBinary, 4, 72)
    target = targetFromBits(bits)

    # TODO other validation of block?  eg timestamp

    if gt(blockHash, 0) && lt(blockHash, target):  #TODO should sgt and slt be used?
        self.saveAncestors(blockHash, hashPrevBlock)

        save(self.block[blockHash]._blockHeader[0], blockHeaderBinary, chars=80) # or 160?

        difficulty = 0x00000000FFFF0000000000000000000000000000000000000000000000000000 / target # https://en.bitcoin.it/wiki/Difficulty
        self.block[blockHash]._score = self.block[hashPrevBlock]._score + difficulty

        if gt(self.block[blockHash]._score, self.highScore):  #TODO use sgt?
            self.heaviestBlock = blockHash
            self.highScore = self.block[blockHash]._score

        return(self.block[blockHash]._height)

    return(0)

# returns 1 if tx is in the block given by 'txBlockHash' and the block is
# in Bitcoin's main chain (ie not a fork)
#
# the merkle proof is represented by 'proofLen', 'hash', 'path', where:
# - 'hash' are the merkle siblings of tx
# - 'path' corresponds to whether the sibling is to the LEFT (1) or RIGHT (2)
# - 'proofLen' is the number of merkle siblings
def verifyTx(tx, proofLen, hash:arr, path:arr, txBlockHash):
    if self.within6Confirms(txBlockHash) || !self.inMainChain(txBlockHash):
        return(0)

    merkle = self.computeMerkle(tx, proofLen, hash, path)
    realMerkleRoot = getMerkleRoot(txBlockHash)

    if merkle == realMerkleRoot:
        return(1)
    else:
        return(0)

# returns the value of processTransaction().  callers should explicitly
# check for a value of 1, since other non-zero values could be error codes
#
# TODO txHash can eventually be computed (dbl sha256 then flip32Bytes) when
# txStr becomes txBinary
def relayTx(txStr:str, txHash, proofLen, hash:arr, path:arr, txBlockHash, contract):
    if self.verifyTx(txHash, proofLen, hash, path, txBlockHash) == 1:
        res = contract.processTransaction(txStr, txHash)
        return(res)
    return(0)

# return the hash of the heaviest block aka the Head
def getBlockchainHead():
    # log(self.heaviestBlock)
    return(self.heaviestBlock)

# return the score of the Head
#
# Because of setPreGenesis(), the score is 1 more than than the
# cumulative difficulty
def getChainScore():
    score = self.block[self.heaviestBlock]._score
    # log(score)
    return(score)

# return -1 if there's an error (eg called with incorrect params)
# [see documentation for verifyTx() for the merkle proof
# format of 'proofLen', 'hash', 'path' ]
def computeMerkle(tx, proofLen, hash:arr, path:arr):
    LEFT_HASH = 1
    RIGHT_HASH = 2

    resultHash = tx
    i = 0
    while i < proofLen:
        proofHex = hash[i]
        if path[i] == LEFT_HASH:
            left = proofHex
            right = resultHash
        elif path[i] == RIGHT_HASH:
            left = resultHash
            right = proofHex

        resultHash = concatHash(left, right)

        i += 1

    if !resultHash:
        return(-1)

    return(resultHash)

# returns 1 if the 'txBlockHash' is within 6 blocks of self.heaviestBlock
# otherwise returns 0.
# note: return value of 0 does not mean 'txBlockHash' has more than 6
# confirmations; a non-existent 'txBlockHash' will lead to a return value of 0
def within6Confirms(txBlockHash):
    blockHash = self.heaviestBlock

    i = 0
    while i < 6:
        if txBlockHash == blockHash:
            return(1)

        # blockHash = self.block[blockHash]._prevBlock
        blockHash = getPrevBlock(blockHash)
        i += 1

    return(0)

# Bitcoin-way of hashing a block header
def fastHashBlock(blockHeaderBinary:str):
    return(flipBytes(sha256(sha256(blockHeaderBinary:str))))

# an owner may transfer/change ownership
def setOwner(newOwner):
    if msg.sender == self.owner:
        self.owner = newOwner
        return(1)
    return(0)

#
# macros
#

# get the parent of '$blockHash'
macro getPrevBlock($blockHash):
    $tmpStr = load(self.block[$blockHash]._blockHeader[0], chars=80)
    getBytesLE($tmpStr, 32, 4)

# get the merkle root of '$blockHash'
macro getMerkleRoot($blockHash):
    $tmpStr = load(self.block[$blockHash]._blockHeader[0], chars=80)
    getBytesLE($tmpStr, 32, 36)

# get $size bytes from $inStr with $offset and return in little endian
macro getBytesLE($inStr, $size, $offset):
    $endIndex = $offset + $size

    $result = 0
    $exponent = 0
    $j = $offset
    while $j < $endIndex:
        $char = getch($inStr, $j)
        # log($char)
        $result += $char * 256^$exponent
        # log(result)

        $j += 1
        $exponent += 1

    $result

# Bitcoin-way of computing the target from the 'bits' field of a blockheader
# based on http://www.righto.com/2014/02/bitcoin-mining-hard-way-algorithms.html#ref3
macro targetFromBits($bits):
    $exp = div($bits, 0x1000000)  # 2^24
    $mant = $bits & 0xffffff
    $mant * 256^($exp - 3)

# Bitcoin-way merkle parent of transaction hashes $tx1 and $tx2
macro concatHash($tx1, $tx2):
    $left = flip32Bytes($tx1)
    $right = flip32Bytes($tx2)

    $hash1 = sha256([$left, $right], chars=64)
    $hash2 = sha256($hash1)

    flip32Bytes($hash2)

# reverse 32 bytes given by '$a'
macro flip32Bytes($a):
    $o = 0
    with $i = 0:
        while $i < 32:
            mstore8(ref($o) + $i, byte(31 - $i, $a))
            $i += 1
    $o
<commit_msg>fix name<commit_after>
inset('btcChain.py')

# btcrelay can relay a transaction to any contract that has a function
# name 'processTransaction' with signature si:i
extern relayDestination: [processTransaction:si:i]

# note: _ancestor[9]
#
# a Bitcoin block (header) is stored as:
# - _blockHeader 80 bytes
# - _height is 1 more than the typical Bitcoin term height/blocknumber [see setPreGensesis()]
# - _score is 1 more than the cumulative difficulty [see setPreGenesis()]
# - _ancestor list for more efficient backtracking (see btcChain)
data block[2^256](_height, _score, _ancestor[9], _blockHeader[])

# block with the highest score (aka the Head of the blockchain)
data heaviestBlock

# highest score among all blocks (so far)
data highScore

# owner/adminstrator of this contract
data owner

def init():
    self.owner = msg.sender
    # TODO anything else to init ?

#TODO for testing only; should be omitted for production
def testingonlySetHeaviest(blockHash):
    if tx.origin == self.owner:
        self.heaviestBlock = blockHash

# this has 2 purposes:
# 1) this should only be called once and is to "store" the block
# with hash zeroes, so that storing the real genesis
# block can be done using storeBlockHeader() instead of a special case
#
# 2) allows testing of storing arbitrary headers and verifying/relaying
# transactions, say from block 300K, instead
# of Satoshi's genesis block which have 0 transactions until much later on
#
# Since zero is the value that indicates non-existence, block _height
# and _score have to be set carefully and they have these consequences:
#
# - _height is 1 more than the typical Bitcoin
# term height/blocknumber. ie genesis has height 1 instead of 0
#
# - _score is 1 more than the cumulative "Bitcoin standard
# difficulty". eg the cumulative difficulty at the block after genesis
# is 2 (it and genesis have difficulty of 1), but this contract assigns
# a score of 3 for the block after genesis [see testStoringHeaders()]
def setPreGenesis(blockHash):
    if tx.origin == self.owner:
        self.heaviestBlock = blockHash

        # _height cannot be set to -1 because inMainChain() assumes that
        # a block with height0 does NOT exist, thus we cannot allow the
        # real genesis block to be at height0
        self.block[blockHash]._height = 0

        # set score to 1, since score0 means block does NOT exist. see check in storeBlockHeader()
        # this means that the score of a block is 1 more than the
        # cumulative difficulty to that block
        self.block[blockHash]._score = 1

        ancLen = self.numAncestorDepths
        i = 0
        while i < ancLen:
            self.block[blockHash]._ancestor[i] = blockHash
            i += 1

# store a Bitcoin block header that must be provided in
# binary format 'blockHeaderBinary'
def storeBlockHeader(blockHeaderBinary:str):
    hashPrevBlock = getBytesLE(blockHeaderBinary, 32, 4)

    assert self.block[hashPrevBlock]._score  # score0 means block does NOT exist; genesis has score of 1

    blockHash = self.fastHashBlock(blockHeaderBinary)

    if self.block[blockHash]._score != 0:  # block already exists
        return(0)

    bits = getBytesLE(blockHeaderBinary, 4, 72)
    target = targetFromBits(bits)

    # TODO other validation of block?  eg timestamp

    if gt(blockHash, 0) && lt(blockHash, target):  #TODO should sgt and slt be used?
        self.saveAncestors(blockHash, hashPrevBlock)

        save(self.block[blockHash]._blockHeader[0], blockHeaderBinary, chars=80) # or 160?

        difficulty = 0x00000000FFFF0000000000000000000000000000000000000000000000000000 / target # https://en.bitcoin.it/wiki/Difficulty
        self.block[blockHash]._score = self.block[hashPrevBlock]._score + difficulty

        if gt(self.block[blockHash]._score, self.highScore):  #TODO use sgt?
            self.heaviestBlock = blockHash
            self.highScore = self.block[blockHash]._score

        return(self.block[blockHash]._height)

    return(0)

# returns 1 if tx is in the block given by 'txBlockHash' and the block is
# in Bitcoin's main chain (ie not a fork)
#
# the merkle proof is represented by 'proofLen', 'hash', 'path', where:
# - 'hash' are the merkle siblings of tx
# - 'path' corresponds to whether the sibling is to the LEFT (1) or RIGHT (2)
# - 'proofLen' is the number of merkle siblings
def verifyTx(tx, proofLen, hash:arr, path:arr, txBlockHash):
    if self.within6Confirms(txBlockHash) || !self.inMainChain(txBlockHash):
        return(0)

    merkle = self.computeMerkle(tx, proofLen, hash, path)
    realMerkleRoot = getMerkleRoot(txBlockHash)

    if merkle == realMerkleRoot:
        return(1)
    else:
        return(0)

# returns the value of processTransaction().  callers should explicitly
# check for a value of 1, since other non-zero values could be error codes
#
# TODO txHash can eventually be computed (dbl sha256 then flip32Bytes) when
# txStr becomes txBinary
def relayTx(txStr:str, txHash, proofLen, hash:arr, path:arr, txBlockHash, contract):
    if self.verifyTx(txHash, proofLen, hash, path, txBlockHash) == 1:
        res = contract.processTransaction(txStr, txHash)
        return(res)
    return(0)

# return the hash of the heaviest block aka the Head
def getBlockchainHead():
    # log(self.heaviestBlock)
    return(self.heaviestBlock)

# return the score of the Head
#
# Because of setPreGenesis(), the score is 1 more than than the
# cumulative difficulty
def getChainScore():
    score = self.block[self.heaviestBlock]._score
    # log(score)
    return(score)

# return -1 if there's an error (eg called with incorrect params)
# [see documentation for verifyTx() for the merkle proof
# format of 'proofLen', 'hash', 'path' ]
def computeMerkle(tx, proofLen, hash:arr, path:arr):
    LEFT_HASH = 1
    RIGHT_HASH = 2

    resultHash = tx
    i = 0
    while i < proofLen:
        proofHex = hash[i]
        if path[i] == LEFT_HASH:
            left = proofHex
            right = resultHash
        elif path[i] == RIGHT_HASH:
            left = resultHash
            right = proofHex

        resultHash = concatHash(left, right)

        i += 1

    if !resultHash:
        return(-1)

    return(resultHash)

# returns 1 if the 'txBlockHash' is within 6 blocks of self.heaviestBlock
# otherwise returns 0.
# note: return value of 0 does not mean 'txBlockHash' has more than 6
# confirmations; a non-existent 'txBlockHash' will lead to a return value of 0
def within6Confirms(txBlockHash):
    blockHash = self.heaviestBlock

    i = 0
    while i < 6:
        if txBlockHash == blockHash:
            return(1)

        # blockHash = self.block[blockHash]._prevBlock
        blockHash = getPrevBlock(blockHash)
        i += 1

    return(0)

# Bitcoin-way of hashing a block header
def fastHashBlock(blockHeaderBinary:str):
    return(flip32Bytes(sha256(sha256(blockHeaderBinary:str))))

# an owner may transfer/change ownership
def setOwner(newOwner):
    if msg.sender == self.owner:
        self.owner = newOwner
        return(1)
    return(0)

#
# macros
#

# get the parent of '$blockHash'
macro getPrevBlock($blockHash):
    $tmpStr = load(self.block[$blockHash]._blockHeader[0], chars=80)
    getBytesLE($tmpStr, 32, 4)

# get the merkle root of '$blockHash'
macro getMerkleRoot($blockHash):
    $tmpStr = load(self.block[$blockHash]._blockHeader[0], chars=80)
    getBytesLE($tmpStr, 32, 36)

# get $size bytes from $inStr with $offset and return in little endian
macro getBytesLE($inStr, $size, $offset):
    $endIndex = $offset + $size

    $result = 0
    $exponent = 0
    $j = $offset
    while $j < $endIndex:
        $char = getch($inStr, $j)
        # log($char)
        $result += $char * 256^$exponent
        # log(result)

        $j += 1
        $exponent += 1

    $result

# Bitcoin-way of computing the target from the 'bits' field of a blockheader
# based on http://www.righto.com/2014/02/bitcoin-mining-hard-way-algorithms.html#ref3
macro targetFromBits($bits):
    $exp = div($bits, 0x1000000)  # 2^24
    $mant = $bits & 0xffffff
    $mant * 256^($exp - 3)

# Bitcoin-way merkle parent of transaction hashes $tx1 and $tx2
macro concatHash($tx1, $tx2):
    $left = flip32Bytes($tx1)
    $right = flip32Bytes($tx2)

    $hash1 = sha256([$left, $right], chars=64)
    $hash2 = sha256($hash1)

    flip32Bytes($hash2)

# reverse 32 bytes given by '$a'
macro flip32Bytes($a):
    $o = 0
    with $i = 0:
        while $i < 32:
            mstore8(ref($o) + $i, byte(31 - $i, $a))
            $i += 1
    $o
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""WebAccount Flask Blueprint"""

from werkzeug.urls import url_unquote
from flask import session, make_response, g, render_template, \
                  request, flash, jsonify, redirect, url_for, current_app
from invenio.sqlalchemyutils import db
from invenio.websession_model import User
from invenio.webinterface_handler_flask_utils import _, InvenioBlueprint
from invenio.config import CFG_SITE_URL, CFG_SITE_SECURE_URL
from invenio.access_control_config import \
     CFG_EXTERNAL_AUTH_USING_SSO, \
     CFG_EXTERNAL_AUTH_LOGOUT_SSO

from invenio.webaccount_forms import LoginForm
from invenio.webuser_flask import login_user, logout_user, current_user

CFG_HAS_HTTPS_SUPPORT = CFG_SITE_SECURE_URL.startswith("https://")
CFG_FULL_HTTPS = CFG_SITE_URL.lower().startswith("https://")

blueprint = InvenioBlueprint('webaccount', __name__,
                url_prefix="/youraccount",
                breadcrumbs=[(_("Your Account"), 'webaccount.index')],
                menubuilder=[('main.personalize', _('Personalize'),
                              'webaccount.display', 3)])

@blueprint.route('/login', methods=['GET', 'POST'])
@blueprint.invenio_set_breadcrumb(_("Login"))
@blueprint.invenio_force_https
def login():
    form = LoginForm(request.values, csrf_enabled=False)
    if form.validate_on_submit():
        try:
            #TODO add login_methods
            user = User.query.filter(db.and_(
                User.nickname == form.nickname.data,
                User.password == form.password.data)).one()
            login_user(user.get_id(), remember_me=form.remember.data)
            flash(_("You are logged in as %s.") % user.nickname, "info")
            # Change HTTP method to https if needed.
            referer = url_unquote(request.form.get("referer",
                                                   url_for(".login")))
            if CFG_FULL_HTTPS or CFG_HAS_HTTPS_SUPPORT and \
                request.url.startswith('https://') and \
                referer.startswith('http://'):
                referer = referer.replace('http://', 'https://', 1)
            return redirect(referer)
        except:
            flash(_("Problem with login."), "error")

    return render_template('webaccount_login.html', form=form)

@blueprint.route('/logout', methods=['GET', 'POST'])
@blueprint.invenio_set_breadcrumb(_("Logout"))
def logout():
    logout_user()
    return render_template('webaccount_logout.html',
                            using_sso=CFG_EXTERNAL_AUTH_USING_SSO,
                            logout_sso=CFG_EXTERNAL_AUTH_LOGOUT_SSO)

def _invenio_settings_plugin_builder(plugin_name, plugin_code):
    """
    Handy function to bridge pluginutils with (Invenio) user settings.
    """
    from invenio.settings import Settings
    if 'settings' in dir(plugin_code):
        candidate = getattr(plugin_code, 'settings')
        if issubclass(candidate, Settings):
            return candidate
    raise ValueError('%s is not a valid settings plugin' % plugin_name)

import os
from invenio.config import CFG_PYLIBDIR
from invenio.pluginutils import PluginContainer
_USER_SETTINGS = PluginContainer(
    os.path.join(CFG_PYLIBDIR, 'invenio', '*_user_settings.py'),
    plugin_builder=_invenio_settings_plugin_builder)

@blueprint.route('/display', methods=['GET', 'POST'])
@blueprint.invenio_authenticated
def index():
    # load plugins
    plugins = [a for a in [s() for (k, s) in _USER_SETTINGS.items()] \
               if a.is_authorized]

    dashboard_settings = current_user.get('dashboard_settings', {})
    order = dashboard_settings.get('order', [])
    plugins = sorted(plugins, key=lambda w: order.index(w.__class__.__name__) \
                            if w.__class__.__name__ in order else len(order))

    return render_template('webaccount_display.html', plugins=plugins)

@blueprint.route('/edit/<name>', methods=['GET', 'POST'])
@blueprint.invenio_set_breadcrumb(_("Edit"))
@blueprint.invenio_authenticated
def edit(name):
    if name not in _USER_SETTINGS:
        flash(_('Invalid plugin name'), 'error')
        return redirect(url_for('.index'))

    plugin = _USER_SETTINGS[name]()
    form = None

    if request.method == 'POST':
        if plugin.form_builder:
            form = plugin.form_builder(request.form)

        if not form or form.validate():
            plugin.store(request.form)
            plugin.save()
            flash(_('Data has been saved.'), 'success')
            return redirect(url_for('.index'))

        flash(_('Please, corrent errors.'), 'error')

    # get post data or load data from settings
    if not form and plugin.form_builder:
        from werkzeug.datastructures import MultiDict
        form = plugin.form_builder(MultiDict(plugin.load()))

    return render_template(getattr(plugin, 'edit_template', '') or \
                           'webaccount_edit.html', plugin=plugin, form=form)
<commit_msg>WebSession: authentication with email address<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""WebAccount Flask Blueprint"""

from werkzeug.urls import url_unquote
from flask import session, make_response, g, render_template, \
                  request, flash, jsonify, redirect, url_for, current_app
from invenio.sqlalchemyutils import db
from invenio.websession_model import User
from invenio.webinterface_handler_flask_utils import _, InvenioBlueprint
from invenio.config import CFG_SITE_URL, CFG_SITE_SECURE_URL
from invenio.access_control_config import \
     CFG_EXTERNAL_AUTH_USING_SSO, \
     CFG_EXTERNAL_AUTH_LOGOUT_SSO

from invenio.webaccount_forms import LoginForm
from invenio.webuser_flask import login_user, logout_user, current_user

CFG_HAS_HTTPS_SUPPORT = CFG_SITE_SECURE_URL.startswith("https://")
CFG_FULL_HTTPS = CFG_SITE_URL.lower().startswith("https://")

blueprint = InvenioBlueprint('webaccount', __name__,
                url_prefix="/youraccount",
                breadcrumbs=[(_("Your Account"), 'webaccount.index')],
                menubuilder=[('main.personalize', _('Personalize'),
                              'webaccount.display', 3)])

@blueprint.route('/login', methods=['GET', 'POST'])
@blueprint.invenio_set_breadcrumb(_("Login"))
@blueprint.invenio_force_https
def login():
    form = LoginForm(request.values, csrf_enabled=False)
    if form.validate_on_submit():
        try:
            #TODO add login_methods
            user = User.query.filter(db.and_(
                db.or_(User.nickname == form.nickname.data,
                       User.email == form.nickname.data),
                User.password == form.password.data)).one()
            login_user(user.get_id(), remember_me=form.remember.data)
            flash(_("You are logged in as %s.") % user.nickname, "info")
            # Change HTTP method to https if needed.
            referer = url_unquote(request.form.get("referer",
                                                   url_for(".login")))
            if CFG_FULL_HTTPS or CFG_HAS_HTTPS_SUPPORT and \
                request.url.startswith('https://') and \
                referer.startswith('http://'):
                referer = referer.replace('http://', 'https://', 1)
            return redirect(referer)
        except:
            flash(_("Problem with login."), "error")

    return render_template('webaccount_login.html', form=form)

@blueprint.route('/logout', methods=['GET', 'POST'])
@blueprint.invenio_set_breadcrumb(_("Logout"))
def logout():
    logout_user()
    flash(_("You have been logged out."), 'info')
    return render_template('webaccount_logout.html',
                            using_sso=CFG_EXTERNAL_AUTH_USING_SSO,
                            logout_sso=CFG_EXTERNAL_AUTH_LOGOUT_SSO)

def _invenio_settings_plugin_builder(plugin_name, plugin_code):
    """
    Handy function to bridge pluginutils with (Invenio) user settings.
    """
    from invenio.settings import Settings
    if 'settings' in dir(plugin_code):
        candidate = getattr(plugin_code, 'settings')
        if issubclass(candidate, Settings):
            return candidate
    raise ValueError('%s is not a valid settings plugin' % plugin_name)

import os
from invenio.config import CFG_PYLIBDIR
from invenio.pluginutils import PluginContainer
_USER_SETTINGS = PluginContainer(
    os.path.join(CFG_PYLIBDIR, 'invenio', '*_user_settings.py'),
    plugin_builder=_invenio_settings_plugin_builder)

@blueprint.route('/display', methods=['GET', 'POST'])
@blueprint.invenio_authenticated
def index():
    # load plugins
    plugins = [a for a in [s() for (k, s) in _USER_SETTINGS.items()] \
               if a.is_authorized]

    dashboard_settings = current_user.get('dashboard_settings', {})
    order = dashboard_settings.get('order', [])
    plugins = sorted(plugins, key=lambda w: order.index(w.__class__.__name__) \
                            if w.__class__.__name__ in order else len(order))

    return render_template('webaccount_display.html', plugins=plugins)

@blueprint.route('/edit/<name>', methods=['GET', 'POST'])
@blueprint.invenio_set_breadcrumb(_("Edit"))
@blueprint.invenio_authenticated
def edit(name):
    if name not in _USER_SETTINGS:
        flash(_('Invalid plugin name'), 'error')
        return redirect(url_for('.index'))

    plugin = _USER_SETTINGS[name]()
    form = None

    if request.method == 'POST':
        if plugin.form_builder:
            form = plugin.form_builder(request.form)

        if not form or form.validate():
            plugin.store(request.form)
            plugin.save()
            flash(_('Data has been saved.'), 'success')
            return redirect(url_for('.index'))

        flash(_('Please, corrent errors.'), 'error')

    # get post data or load data from settings
    if not form and plugin.form_builder:
        from werkzeug.datastructures import MultiDict
        form = plugin.form_builder(MultiDict(plugin.load()))

    return render_template(getattr(plugin, 'edit_template', '') or \
                           'webaccount_edit.html', plugin=plugin, form=form)
<|endoftext|>