<commit_before># -*- coding: utf-8 -*-
# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
minio.helpers

This module implements all helper functions.

:copyright: (c) 2015 by Minio, Inc.
:license: Apache 2.0, see LICENSE for more details.

"""

from __future__ import absolute_import
import io

import collections
import binascii
import hashlib
import re
import os
import errno
import math

from .compat import urlsplit, urlencode
from .error import (InvalidBucketError, InvalidEndpointError,
                    InvalidArgumentError)

# Constants
MAX_MULTIPART_COUNT = 10000 # 10000 parts
MAX_MULTIPART_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 * 1024  # 5TiB
MIN_OBJECT_SIZE = 5 * 1024 * 1024  # 5MiB

_VALID_BUCKETNAME_REGEX = re.compile('^[a-z0-9][a-z0-9\\.\\-]+[a-z0-9]$')
_ALLOWED_HOSTNAME_REGEX = re.compile(
    '^((?!-)[A-Z\\d-]{1,63}(?<!-)\\.)*((?!-)[A-Z\\d-]{1,63}(?<!-))$',
    re.IGNORECASE)

def dump_http(method, url, request_headers, response, output_stream):
    """
    Dump all headers and response headers into output_stream.

    :param request_headers: Dictionary of HTTP request headers.
    :param response_headers: Dictionary of HTTP response headers.
    :param output_stream: Stream where the request is being dumped at.
    """

    # Start header.
    output_stream.write('---------START-HTTP---------\n')

    # Get parsed url.
    parsed_url = urlsplit(url)

    # Dump all request headers recursively.
    http_path = parsed_url.path
    if parsed_url.query:
        http_path = http_path + '?' + parsed_url.query

    output_stream.write('{0} {1} HTTP/1.1\n'.format(method,
                                                    http_path))

    for k, v in list(request_headers.items()):
        if k is 'authorization':
            # Redact signature header value from trace logs.
            v = re.sub(r'Signature=([[0-9a-f]+)', 'Signature=*REDACTED*', v)
        output_stream.write('{0}: {1}\n'.format(k.title(), v))

    # Write a new line.
    output_stream.write('\n')

    # Write response status code.
    output_stream.write('HTTP/1.1 {0}\n'.format(response.status))

    # Dump all response headers recursively.
    for k, v in list(response.getheaders().items()):
        output_stream.write('{0}: {1}\n'.format(k.title(), v))

    # For all errors write all the available response body.
    if response.status != 200 and \
       response.status != 204 and response.status != 206:
        output_stream.write('{0}'.format(response.read()))

    # End header.
    output_stream.write('---------END-HTTP---------\n')

def mkdir_p(path):
    """
    Recursively creates parent and sub directories.

    :param path:
    """
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

class PartMetadata(object):
    """
    Parts manager split parts metadata :class:`PartMetadata <PartMetadata>`.

    :param data: Part writer object backed by temporary file.
    :param md5digest: Md5sum digest of the part.
    :param sha256digest: Sha256sum digest of the part.
    :param size: Size of the part.
    """
    def __init__(self, data, md5digest, sha256digest, size):
        self.data = data
        self.md5digest = md5digest
        self.sha256digest = sha256digest
        self.size = size

def parts_manager(data, part_size=5*1024*1024):
    """
    Reads data and provides temporary files of a given size.

    :param data: Input reader object which needs to be saved.
    :param part_size: Individual part number defaults to 5MB.
    :return: Returns :class:`PartMetadata <PartMetadata>`
    """
    tmpdata = io.BytesIO()
    md5hasher = hashlib.md5()
    sha256hasher = hashlib.sha256()
    total_read = 0
    while total_read < part_size:
        current_data = data.read(1024)
        if not current_data or len(current_data) == 0:
            break
        tmpdata.write(current_data)
        md5hasher.update(current_data)
        sha256hasher.update(current_data)
        total_read = total_read + len(current_data)

    return PartMetadata(tmpdata, md5hasher.digest(),
                        sha256hasher.digest(), total_read)

def ignore_headers(headers_to_sign):
    """
    Ignore headers.
    """
    # Excerpts from @lsegal -
    # https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
    #
    #  User-Agent:
    #
    #      This is ignored from signing because signing this causes problems
    #      with generating pre-signed URLs (that are executed by other agents)
    #      or when customers pass requests through proxies, which may modify
    #      the user-agent.
    #
    #  Content-Length:
    #
    #      This is ignored from signing because generating a pre-signed URL
    #      should not provide a content-length constraint, specifically when
    #      vending a S3 pre-signed PUT URL. The corollary to this is that when
    #      sending regular requests (non-pre-signed), the signature contains
    #      a checksum of the body, which implicitly validates the payload
    #      length (since changing the number of bytes would change the
    #      checksum) and therefore this header is not valuable in the
    #      signature.
    #
    #  Content-Type:
    #
    #      Signing this header causes quite a number of problems in browser
    #      environments, where browsers like to modify and normalize the
    #      content-type header in different ways. There is more information
    #      on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding
    #      this field simplifies logic and reduces the possibility of bugs.
    #
    #  Authorization:
    #
    #      Is skipped for obvious reasons
    ignored_headers = ['Authorization', 'Content-Length',
                       'Content-Type', 'User-Agent']
    for ignored_header in ignored_headers:
        if ignored_header in headers_to_sign:
            del headers_to_sign[ignored_header]

    return headers_to_sign

AWS_S3_ENDPOINT_MAP = {
    'us-east-1': 's3.amazonaws.com',
    'us-west-2': 's3-us-west-2.amazonaws.com',
    'us-west-1': 's3-us-west-1.amazonaws.com',
    'eu-west-1': 's3-eu-west-1.amazonaws.com',
    'sa-east-1': 's3-sa-east-1.amazonaws.com',
    'eu-central-1': 's3-eu-central-1.amazonaws.com',
    'ap-southeast-1': 's3-ap-southeast-1.amazonaws.com',
    'ap-northeast-1': 's3-ap-northeast-1.amazonaws.com',
    'ap-northeast-2': 's3-ap-northeast-2.amazonaws.com',
}

def get_s3_endpoint(region):
    if region in AWS_S3_ENDPOINT_MAP:
        return AWS_S3_ENDPOINT_MAP[region]
    return 's3.amazonaws.com'

def get_target_url(endpoint_url, bucket_name=None, object_name=None,
                   bucket_region='us-east-1', query=None):
    """
    Construct final target url.

    :param endpoint_url: Target endpoint url where request is served to.
    :param bucket_name: Bucket component for the target url.
    :param object_name: Object component for the target url.
    :param bucket_region: Bucket region for the target url.
    :param query: Query parameters as a *dict* for the target url.
    :return: Returns final target url as *str*.
    """
    # New url
    url = None

    # Parse url
    parsed_url = urlsplit(endpoint_url)

    # Get new host, scheme.
    host = parsed_url.netloc
    if 's3.amazonaws.com' in host:
        host = get_s3_endpoint(bucket_region)
    scheme = parsed_url.scheme

    url = scheme + '://' + host
    if bucket_name:
        # Save if target url will have buckets which suppport
        # virtual host.
        is_virtual_host_style = is_virtual_host(endpoint_url,
                                                bucket_name)
        if is_virtual_host_style:
            url = (scheme + '://' + bucket_name + '.' + host)
        else:
            url = (scheme + '://' + host + '/' + bucket_name)

    url_components = [url]
    url_components.append('/')

    if object_name:
        object_name = encode_object_name(object_name)
        url_components.append(object_name)

    if query:
        ordered_query = collections.OrderedDict(sorted(query.items()))
        query_components = []
        for component_key in ordered_query:
            single_component = [component_key]
            if ordered_query[component_key] is not None:
                single_component.append('=')
                encoded_query = urlencode(
                    str(ordered_query[component_key])).replace(
                        '/',
                        '%2F')
                single_component.append(encoded_query)
            query_components.append(''.join(single_component))

        query_string = '&'.join(query_components)
        if query_string:
            url_components.append('?')
            url_components.append(query_string)

    return ''.join(url_components)

def is_valid_endpoint(endpoint):
    """
    Verify if endpoint is valid.

    :type endpoint: string
    :param endpoint: An endpoint. Must have at least a scheme and a hostname.
    :return: True if the endpoint is valid. Raise :exc:`InvalidEndpointError`
       otherwise.
    """
    try:
        if urlsplit(endpoint).scheme:
            raise InvalidEndpointError('Hostname cannot have a scheme.')

        hostname = endpoint.split(':')[0]
        if hostname is None:
            raise InvalidEndpointError('Hostname cannot be empty.')

        if len(hostname) > 255:
            raise InvalidEndpointError('Hostname cannot be greater than 255.')

        if hostname[-1] == '.':
            hostname = hostname[:-1]

        if not _ALLOWED_HOSTNAME_REGEX.match(hostname):
            raise InvalidEndpointError('Hostname does not meet URL standards.')

        if hostname.endswith('.amazonaws.com') and \
           (hostname != 's3.amazonaws.com'):
            raise InvalidEndpointError('Amazon S3 hostname should be '
                                       's3.amazonaws.com.')
    except AttributeError as error:
        raise TypeError(error)

    return True

def is_virtual_host(endpoint_url, bucket_name):
    """
    Check to see if the ``bucket_name`` can be part of virtual host
    style.

    :param endpoint_url: Endpoint url which will be used for virtual host.
    :param bucket_name: Bucket name to be validated against.
    """
    is_valid_bucket_name(bucket_name)

    parsed_url = urlsplit(endpoint_url)
    # bucket_name can be valid but '.' in the hostname will fail
    # SSL certificate validation. So do not use host-style for
    # such buckets.
    if 'https' in parsed_url.scheme and '.' in bucket_name:
        return False
    if 's3.amazonaws.com' in parsed_url.netloc:
        return True
    return False

def is_valid_bucket_name(bucket_name):
    """
    Check to see if the ``bucket_name`` complies with the
    restricted DNS naming conventions necessary to allow
    access via virtual-hosting style.

    :param bucket_name: Bucket name in *str*.
    :return: True if the bucket is valid. Raise :exc:`InvalidBucketError`
       otherwise.
    """
    # Verify bucket name length.
    if len(bucket_name) < 3:
        raise InvalidBucketError('Bucket name cannot be less than'
                                 ' 3 characters.')
    if len(bucket_name) > 63:
        raise InvalidBucketError('Bucket name cannot be more than'
                                 ' 63 characters.')
    if '..' in bucket_name:
        raise InvalidBucketError('Bucket name cannot have successive'
                                 ' periods.')

    match = _VALID_BUCKETNAME_REGEX.match(bucket_name)
    if match is None or match.end() != len(bucket_name):
        raise InvalidBucketError('Bucket name does not follow S3 standards.'
                                 ' Bucket: {0}'.format(bucket_name))

    return True

def is_non_empty_string(input_string):
    """
    Validate if non empty string

    :param input_string: Input is a *str*.
    :return: True if input is string and non empty.
       Raise :exc:`Exception` otherwise.
    """
    try:
        if not input_string.strip():
            raise ValueError()
    except AttributeError as error:
        raise TypeError(error)

    return True

def encode_object_name(object_name):
    """
    URL encode input object name.

    :param object_name: Un-encoded object name.
    :return: URL encoded input object name.
    """
    is_non_empty_string(object_name)
    return urlencode(object_name)

def get_sha256(content):
    """
    Calculate sha256 digest of input byte array.

    :param content: Input byte array.
    :return: sha256 digest of input byte array.
    """
    if len(content) == 0:
        content = b''
    hasher = hashlib.sha256()
    hasher.update(content)
    return hasher.digest()

def get_md5(content):
    """
    Calculate md5 digest of input byte array.

    :param content: Input byte array.
    :return: md5 digest of input byte array.
    """
    if len(content) == 0:
        content = b''
    hasher = hashlib.md5()
    hasher.update(content)
    return hasher.digest()

def encode_to_base64(content):
    """
    Calculate base64 of input byte array.

    :param content: Input byte array.
    :return: base64 encoding of input byte array.
    """
    return binascii.b2a_base64(content).strip().decode('utf-8')

def encode_to_hex(content):
    """
    Calculate hex for input byte array.

    :param content: Input byte array.
    :return: hexlified input byte array.
    """
    return binascii.hexlify(content)

def optimal_part_info(length):
    """
    Calculate optimal part size for multipart uploads.

    :param length: Input length to calculate part size of.
    :return: Optimal part size.
    """
    # object size is '-1' set it to 5TiB.
    if length == -1:
        length = MAX_MULTIPART_OBJECT_SIZE
    if length > MAX_MULTIPART_OBJECT_SIZE:
        raise InvalidArgumentError('Input content size is bigger '
                                   ' than allowed maximum of 5TiB.')

    # Use floats for part size for all calculations to avoid
    # overflows during float64 to int64 conversions.
    part_size_float = math.ceil(length/MAX_MULTIPART_COUNT)
    part_size_float = (math.ceil(part_size_float/MIN_OBJECT_SIZE)
                       * MIN_OBJECT_SIZE)
    # Total parts count.
    total_parts_count = int(math.ceil(length/part_size_float))
    # Part size.
    part_size = int(part_size_float)
    # Last part size.
    last_part_size = length - int(total_parts_count-1)*part_size
    return total_parts_count, part_size, last_part_size
<commit_msg>Code:helpers.py/Added ap-south-1 region to the list of available s3 regions. (#397)<commit_after># -*- coding: utf-8 -*-
# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
minio.helpers

This module implements all helper functions.

:copyright: (c) 2015 by Minio, Inc.
:license: Apache 2.0, see LICENSE for more details.

"""

from __future__ import absolute_import
import io

import collections
import binascii
import hashlib
import re
import os
import errno
import math

from .compat import urlsplit, urlencode
from .error import (InvalidBucketError, InvalidEndpointError,
                    InvalidArgumentError)

# Constants
MAX_MULTIPART_COUNT = 10000 # 10000 parts
MAX_MULTIPART_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 * 1024  # 5TiB
MIN_OBJECT_SIZE = 5 * 1024 * 1024  # 5MiB

_VALID_BUCKETNAME_REGEX = re.compile('^[a-z0-9][a-z0-9\\.\\-]+[a-z0-9]$')
_ALLOWED_HOSTNAME_REGEX = re.compile(
    '^((?!-)[A-Z\\d-]{1,63}(?<!-)\\.)*((?!-)[A-Z\\d-]{1,63}(?<!-))$',
    re.IGNORECASE)

def dump_http(method, url, request_headers, response, output_stream):
    """
    Dump all headers and response headers into output_stream.

    :param request_headers: Dictionary of HTTP request headers.
    :param response_headers: Dictionary of HTTP response headers.
    :param output_stream: Stream where the request is being dumped at.
    """

    # Start header.
    output_stream.write('---------START-HTTP---------\n')

    # Get parsed url.
    parsed_url = urlsplit(url)

    # Dump all request headers recursively.
    http_path = parsed_url.path
    if parsed_url.query:
        http_path = http_path + '?' + parsed_url.query

    output_stream.write('{0} {1} HTTP/1.1\n'.format(method,
                                                    http_path))

    for k, v in list(request_headers.items()):
        if k is 'authorization':
            # Redact signature header value from trace logs.
            v = re.sub(r'Signature=([[0-9a-f]+)', 'Signature=*REDACTED*', v)
        output_stream.write('{0}: {1}\n'.format(k.title(), v))

    # Write a new line.
    output_stream.write('\n')

    # Write response status code.
    output_stream.write('HTTP/1.1 {0}\n'.format(response.status))

    # Dump all response headers recursively.
    for k, v in list(response.getheaders().items()):
        output_stream.write('{0}: {1}\n'.format(k.title(), v))

    # For all errors write all the available response body.
    if response.status != 200 and \
       response.status != 204 and response.status != 206:
        output_stream.write('{0}'.format(response.read()))

    # End header.
    output_stream.write('---------END-HTTP---------\n')

def mkdir_p(path):
    """
    Recursively creates parent and sub directories.

    :param path:
    """
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

class PartMetadata(object):
    """
    Parts manager split parts metadata :class:`PartMetadata <PartMetadata>`.

    :param data: Part writer object backed by temporary file.
    :param md5digest: Md5sum digest of the part.
    :param sha256digest: Sha256sum digest of the part.
    :param size: Size of the part.
    """
    def __init__(self, data, md5digest, sha256digest, size):
        self.data = data
        self.md5digest = md5digest
        self.sha256digest = sha256digest
        self.size = size

def parts_manager(data, part_size=5*1024*1024):
    """
    Reads data and provides temporary files of a given size.

    :param data: Input reader object which needs to be saved.
    :param part_size: Individual part number defaults to 5MB.
    :return: Returns :class:`PartMetadata <PartMetadata>`
    """
    tmpdata = io.BytesIO()
    md5hasher = hashlib.md5()
    sha256hasher = hashlib.sha256()
    total_read = 0
    while total_read < part_size:
        current_data = data.read(1024)
        if not current_data or len(current_data) == 0:
            break
        tmpdata.write(current_data)
        md5hasher.update(current_data)
        sha256hasher.update(current_data)
        total_read = total_read + len(current_data)

    return PartMetadata(tmpdata, md5hasher.digest(),
                        sha256hasher.digest(), total_read)

def ignore_headers(headers_to_sign):
    """
    Ignore headers.
    """
    # Excerpts from @lsegal -
    # https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
    #
    #  User-Agent:
    #
    #      This is ignored from signing because signing this causes problems
    #      with generating pre-signed URLs (that are executed by other agents)
    #      or when customers pass requests through proxies, which may modify
    #      the user-agent.
    #
    #  Content-Length:
    #
    #      This is ignored from signing because generating a pre-signed URL
    #      should not provide a content-length constraint, specifically when
    #      vending a S3 pre-signed PUT URL. The corollary to this is that when
    #      sending regular requests (non-pre-signed), the signature contains
    #      a checksum of the body, which implicitly validates the payload
    #      length (since changing the number of bytes would change the
    #      checksum) and therefore this header is not valuable in the
    #      signature.
    #
    #  Content-Type:
    #
    #      Signing this header causes quite a number of problems in browser
    #      environments, where browsers like to modify and normalize the
    #      content-type header in different ways. There is more information
    #      on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding
    #      this field simplifies logic and reduces the possibility of bugs.
    #
    #  Authorization:
    #
    #      Is skipped for obvious reasons
    ignored_headers = ['Authorization', 'Content-Length',
                       'Content-Type', 'User-Agent']
    for ignored_header in ignored_headers:
        if ignored_header in headers_to_sign:
            del headers_to_sign[ignored_header]

    return headers_to_sign

AWS_S3_ENDPOINT_MAP = {
    'us-east-1': 's3.amazonaws.com',
    'us-west-2': 's3-us-west-2.amazonaws.com',
    'us-west-1': 's3-us-west-1.amazonaws.com',
    'eu-west-1': 's3-eu-west-1.amazonaws.com',
    'sa-east-1': 's3-sa-east-1.amazonaws.com',
    'eu-central-1': 's3-eu-central-1.amazonaws.com',
    'ap-south-1': 's3.ap-south-1.amazonaws.com',
    'ap-southeast-1': 's3-ap-southeast-1.amazonaws.com',
    'ap-northeast-1': 's3-ap-northeast-1.amazonaws.com',
    'ap-northeast-2': 's3-ap-northeast-2.amazonaws.com',
}

def get_s3_endpoint(region):
    if region in AWS_S3_ENDPOINT_MAP:
        return AWS_S3_ENDPOINT_MAP[region]
    return 's3.amazonaws.com'

def get_target_url(endpoint_url, bucket_name=None, object_name=None,
                   bucket_region='us-east-1', query=None):
    """
    Construct final target url.

    :param endpoint_url: Target endpoint url where request is served to.
    :param bucket_name: Bucket component for the target url.
    :param object_name: Object component for the target url.
    :param bucket_region: Bucket region for the target url.
    :param query: Query parameters as a *dict* for the target url.
    :return: Returns final target url as *str*.
    """
    # New url
    url = None

    # Parse url
    parsed_url = urlsplit(endpoint_url)

    # Get new host, scheme.
    host = parsed_url.netloc
    if 's3.amazonaws.com' in host:
        host = get_s3_endpoint(bucket_region)
    scheme = parsed_url.scheme

    url = scheme + '://' + host
    if bucket_name:
        # Save if target url will have buckets which suppport
        # virtual host.
        is_virtual_host_style = is_virtual_host(endpoint_url,
                                                bucket_name)
        if is_virtual_host_style:
            url = (scheme + '://' + bucket_name + '.' + host)
        else:
            url = (scheme + '://' + host + '/' + bucket_name)

    url_components = [url]
    url_components.append('/')

    if object_name:
        object_name = encode_object_name(object_name)
        url_components.append(object_name)

    if query:
        ordered_query = collections.OrderedDict(sorted(query.items()))
        query_components = []
        for component_key in ordered_query:
            single_component = [component_key]
            if ordered_query[component_key] is not None:
                single_component.append('=')
                encoded_query = urlencode(
                    str(ordered_query[component_key])).replace(
                        '/',
                        '%2F')
                single_component.append(encoded_query)
            query_components.append(''.join(single_component))

        query_string = '&'.join(query_components)
        if query_string:
            url_components.append('?')
            url_components.append(query_string)

    return ''.join(url_components)

def is_valid_endpoint(endpoint):
    """
    Verify if endpoint is valid.

    :type endpoint: string
    :param endpoint: An endpoint. Must have at least a scheme and a hostname.
    :return: True if the endpoint is valid. Raise :exc:`InvalidEndpointError`
       otherwise.
    """
    try:
        if urlsplit(endpoint).scheme:
            raise InvalidEndpointError('Hostname cannot have a scheme.')

        hostname = endpoint.split(':')[0]
        if hostname is None:
            raise InvalidEndpointError('Hostname cannot be empty.')

        if len(hostname) > 255:
            raise InvalidEndpointError('Hostname cannot be greater than 255.')

        if hostname[-1] == '.':
            hostname = hostname[:-1]

        if not _ALLOWED_HOSTNAME_REGEX.match(hostname):
            raise InvalidEndpointError('Hostname does not meet URL standards.')

        if hostname.endswith('.amazonaws.com') and \
           (hostname != 's3.amazonaws.com'):
            raise InvalidEndpointError('Amazon S3 hostname should be '
                                       's3.amazonaws.com.')
    except AttributeError as error:
        raise TypeError(error)

    return True

def is_virtual_host(endpoint_url, bucket_name):
    """
    Check to see if the ``bucket_name`` can be part of virtual host
    style.

    :param endpoint_url: Endpoint url which will be used for virtual host.
    :param bucket_name: Bucket name to be validated against.
    """
    is_valid_bucket_name(bucket_name)

    parsed_url = urlsplit(endpoint_url)
    # bucket_name can be valid but '.' in the hostname will fail
    # SSL certificate validation. So do not use host-style for
    # such buckets.
    if 'https' in parsed_url.scheme and '.' in bucket_name:
        return False
    if 's3.amazonaws.com' in parsed_url.netloc:
        return True
    return False

def is_valid_bucket_name(bucket_name):
    """
    Check to see if the ``bucket_name`` complies with the
    restricted DNS naming conventions necessary to allow
    access via virtual-hosting style.

    :param bucket_name: Bucket name in *str*.
    :return: True if the bucket is valid. Raise :exc:`InvalidBucketError`
       otherwise.
    """
    # Verify bucket name length.
    if len(bucket_name) < 3:
        raise InvalidBucketError('Bucket name cannot be less than'
                                 ' 3 characters.')
    if len(bucket_name) > 63:
        raise InvalidBucketError('Bucket name cannot be more than'
                                 ' 63 characters.')
    if '..' in bucket_name:
        raise InvalidBucketError('Bucket name cannot have successive'
                                 ' periods.')

    match = _VALID_BUCKETNAME_REGEX.match(bucket_name)
    if match is None or match.end() != len(bucket_name):
        raise InvalidBucketError('Bucket name does not follow S3 standards.'
                                 ' Bucket: {0}'.format(bucket_name))

    return True

def is_non_empty_string(input_string):
    """
    Validate if non empty string

    :param input_string: Input is a *str*.
    :return: True if input is string and non empty.
       Raise :exc:`Exception` otherwise.
    """
    try:
        if not input_string.strip():
            raise ValueError()
    except AttributeError as error:
        raise TypeError(error)

    return True

def encode_object_name(object_name):
    """
    URL encode input object name.

    :param object_name: Un-encoded object name.
    :return: URL encoded input object name.
    """
    is_non_empty_string(object_name)
    return urlencode(object_name)

def get_sha256(content):
    """
    Calculate sha256 digest of input byte array.

    :param content: Input byte array.
    :return: sha256 digest of input byte array.
    """
    if len(content) == 0:
        content = b''
    hasher = hashlib.sha256()
    hasher.update(content)
    return hasher.digest()

def get_md5(content):
    """
    Calculate md5 digest of input byte array.

    :param content: Input byte array.
    :return: md5 digest of input byte array.
    """
    if len(content) == 0:
        content = b''
    hasher = hashlib.md5()
    hasher.update(content)
    return hasher.digest()

def encode_to_base64(content):
    """
    Calculate base64 of input byte array.

    :param content: Input byte array.
    :return: base64 encoding of input byte array.
    """
    return binascii.b2a_base64(content).strip().decode('utf-8')

def encode_to_hex(content):
    """
    Calculate hex for input byte array.

    :param content: Input byte array.
    :return: hexlified input byte array.
    """
    return binascii.hexlify(content)

def optimal_part_info(length):
    """
    Calculate optimal part size for multipart uploads.

    :param length: Input length to calculate part size of.
    :return: Optimal part size.
    """
    # object size is '-1' set it to 5TiB.
    if length == -1:
        length = MAX_MULTIPART_OBJECT_SIZE
    if length > MAX_MULTIPART_OBJECT_SIZE:
        raise InvalidArgumentError('Input content size is bigger '
                                   ' than allowed maximum of 5TiB.')

    # Use floats for part size for all calculations to avoid
    # overflows during float64 to int64 conversions.
    part_size_float = math.ceil(length/MAX_MULTIPART_COUNT)
    part_size_float = (math.ceil(part_size_float/MIN_OBJECT_SIZE)
                       * MIN_OBJECT_SIZE)
    # Total parts count.
    total_parts_count = int(math.ceil(length/part_size_float))
    # Part size.
    part_size = int(part_size_float)
    # Last part size.
    last_part_size = length - int(total_parts_count-1)*part_size
    return total_parts_count, part_size, last_part_size
<|endoftext|>


<commit_before>try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error

from twitter.twitter_globals import POST_ACTIONS
from twitter.auth import NoAuth

try:
    import json
except ImportError:
    import simplejson as json

class _DEFAULT(object):
    pass

class TwitterError(Exception):
    """
    Base Exception thrown by the Twitter object when there is a
    general error interacting with the API.
    """
    pass

class TwitterHTTPError(TwitterError):
    """
    Exception thrown by the Twitter object when there is an
    HTTP error interacting with twitter.com.
    """
    def __init__(self, e, uri, format, uriparts):
        self.e = e
        self.uri = uri
        self.format = format
        self.uriparts = uriparts

    def __str__(self):
        return (
            "Twitter sent status %i for URL: %s.%s using parameters: "
            "(%s)\ndetails: %s" %(
                self.e.code, self.uri, self.format, self.uriparts,
                self.e.fp.read()))

class TwitterResponse(object):
    """
    Response from a twitter request. Behaves like a list or a string
    (depending on requested format) but it has a few other interesting
    attributes.

    `headers` gives you access to the response headers as an
    httplib.HTTPHeaders instance. You can do
    `response.headers.getheader('h')` to retrieve a header.
    """
    def __init__(self, headers):
        self.headers = headers

    @property
    def rate_limit_remaining(self):
        """
        Remaining requests in the current rate-limit.
        """
        return int(self.headers.getheader('X-RateLimit-Remaining'))

    @property
    def rate_limit_reset(self):
        """
        Time in UTC epoch seconds when the rate limit will reset.
        """
        return int(self.headers.getheader('X-RateLimit-Reset'))

def wrap_response(response, headers):
    response_typ = type(response)
    if response_typ is bool:
        # HURF DURF MY NAME IS PYTHON AND I CAN'T SUBCLASS bool.
        response_typ = int

    class WrappedTwitterResponse(response_typ, TwitterResponse):
        __doc__ = TwitterResponse.__doc__

        def __init__(self, response):
            if response_typ is not int:
                response_typ.__init__(self, response)
            TwitterResponse.__init__(self, headers)

    return WrappedTwitterResponse(response)

class TwitterCall(object):

    def __init__(
        self, auth, format, domain, callable_cls, uri="",
        uriparts=None, secure=True):
        self.auth = auth
        self.format = format
        self.domain = domain
        self.callable_cls = callable_cls
        self.uri = uri
        self.uriparts = uriparts
        self.secure = secure

    def __getattr__(self, k):
        try:
            return object.__getattr__(self, k)
        except AttributeError:
            def extend_call(arg):
                return self.callable_cls(
                    auth=self.auth, format=self.format, domain=self.domain,
                    callable_cls=self.callable_cls, uriparts=self.uriparts \
                        + (arg,),
                    secure=self.secure)
            if k == "_":
                return extend_call
            else:
                return extend_call(k)

    def __call__(self, **kwargs):
        # Build the uri.
        uriparts = []
        for uripart in self.uriparts:
            # If this part matches a keyword argument, use the
            # supplied value otherwise, just use the part.
            uriparts.append(str(kwargs.pop(uripart, uripart)))
        uri = '/'.join(uriparts)

        method = "GET"
        for action in POST_ACTIONS:
            if uri.endswith(action):
                method = "POST"
                break

        # If an id kwarg is present and there is no id to fill in in
        # the list of uriparts, assume the id goes at the end.
        id = kwargs.pop('id', None)
        if id:
            uri += "/%s" %(id)

        secure_str = ''
        if self.secure:
            secure_str = 's'
        dot = ""
        if self.format:
            dot = "."
        uriBase = "http%s://%s/%s%s%s" %(
                    secure_str, self.domain, uri, dot, self.format)

        headers = {}
        if self.auth:
            headers.update(self.auth.generate_headers())
            arg_data = self.auth.encode_params(uriBase, method, kwargs)
            if method == 'GET':
                uriBase += '?' + arg_data
                body = None
            else:
                body = arg_data.encode('utf8')

        req = urllib_request.Request(uriBase, body, headers)
        return self._handle_response(req, uri, arg_data)

    def _handle_response(self, req, uri, arg_data):
        try:
            handle = urllib_request.urlopen(req)
            if "json" == self.format:
                res = json.loads(handle.read().decode('utf8'))
                return wrap_response(res, handle.headers)
            else:
                return wrap_response(
                    handle.read().decode('utf8'), handle.headers)
        except urllib_error.HTTPError as e:
            if (e.code == 304):
                return []
            else:
                raise TwitterHTTPError(e, uri, self.format, arg_data)

class Twitter(TwitterCall):
    """
    The minimalist yet fully featured Twitter API class.

    Get RESTful data by accessing members of this class. The result
    is decoded python objects (lists and dicts).

    The Twitter API is documented here:

      http://dev.twitter.com/doc

    Examples::

      twitter = Twitter(
          auth=OAuth(token, token_key, con_secret, con_secret_key)))

      # Get the public timeline
      twitter.statuses.public_timeline()

      # Get a particular friend's timeline
      twitter.statuses.friends_timeline(id="billybob")

      # Also supported (but totally weird)
      twitter.statuses.friends_timeline.billybob()

      # Send a direct message
      twitter.direct_messages.new(
          user="billybob",
          text="I think yer swell!")

      # Get the members of a particular list of a particular friend
      twitter.user.listname.members(user="billybob", listname="billysbuds")

    Searching Twitter::

      twitter_search = Twitter(domain="search.twitter.com")

      # Find the latest search trends
      twitter_search.trends()

      # Search for the latest News on #gaza
      twitter_search.search(q="#gaza")

    Using the data returned
    -----------------------

    Twitter API calls return decoded JSON. This is converted into
    a bunch of Python lists, dicts, ints, and strings. For example::

      x = twitter.statuses.public_timeline()

      # The first 'tweet' in the timeline
      x[0]

      # The screen name of the user who wrote the first 'tweet'
      x[0]['user']['screen_name']

    Getting raw XML data
    --------------------

    If you prefer to get your Twitter data in XML format, pass
    format="xml" to the Twitter object when you instantiate it::

      twitter = Twitter(format="xml")

      The output will not be parsed in any way. It will be a raw string
      of XML.

    """
    def __init__(
        self, format="json",
        domain="api.twitter.com", secure=True, auth=None,
        api_version=_DEFAULT):
        """
        Create a new twitter API connector.

        Pass an `auth` parameter to use the credentials of a specific
        user. Generally you'll want to pass an `OAuth`
        instance::

            twitter = Twitter(auth=OAuth(
                    token, token_secret, consumer_key, consumer_secret))

        `domain` lets you change the domain you are connecting. By
        default it's `api.twitter.com` but `search.twitter.com` may be
        useful too.

        If `secure` is False you will connect with HTTP instead of
        HTTPS.

        `api_version` is used to set the base uri. By default it's
        '1'. If you are using "search.twitter.com" set this to None.
        """
        if not auth:
            auth = NoAuth()

        if (format not in ("json", "xml", "")):
            raise ValueError("Unknown data format '%s'" %(format))

        if api_version is _DEFAULT:
            if domain == 'api.twitter.com':
                api_version = '1'
            else:
                api_version = None

        uriparts = ()
        if api_version:
            uriparts += (str(api_version),)

        TwitterCall.__init__(
            self, auth=auth, format=format, domain=domain,
            callable_cls=TwitterCall,
            secure=secure, uriparts=uriparts)

__all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]
<commit_msg>Who knew this mysterious looking code was totally useless? Not me.<commit_after>try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error

from twitter.twitter_globals import POST_ACTIONS
from twitter.auth import NoAuth

try:
    import json
except ImportError:
    import simplejson as json

class _DEFAULT(object):
    pass

class TwitterError(Exception):
    """
    Base Exception thrown by the Twitter object when there is a
    general error interacting with the API.
    """
    pass

class TwitterHTTPError(TwitterError):
    """
    Exception thrown by the Twitter object when there is an
    HTTP error interacting with twitter.com.
    """
    def __init__(self, e, uri, format, uriparts):
        self.e = e
        self.uri = uri
        self.format = format
        self.uriparts = uriparts

    def __str__(self):
        return (
            "Twitter sent status %i for URL: %s.%s using parameters: "
            "(%s)\ndetails: %s" %(
                self.e.code, self.uri, self.format, self.uriparts,
                self.e.fp.read()))

class TwitterResponse(object):
    """
    Response from a twitter request. Behaves like a list or a string
    (depending on requested format) but it has a few other interesting
    attributes.

    `headers` gives you access to the response headers as an
    httplib.HTTPHeaders instance. You can do
    `response.headers.getheader('h')` to retrieve a header.
    """
    def __init__(self, headers):
        self.headers = headers

    @property
    def rate_limit_remaining(self):
        """
        Remaining requests in the current rate-limit.
        """
        return int(self.headers.getheader('X-RateLimit-Remaining'))

    @property
    def rate_limit_reset(self):
        """
        Time in UTC epoch seconds when the rate limit will reset.
        """
        return int(self.headers.getheader('X-RateLimit-Reset'))

def wrap_response(response, headers):
    response_typ = type(response)
    if response_typ is bool:
        # HURF DURF MY NAME IS PYTHON AND I CAN'T SUBCLASS bool.
        response_typ = int

    class WrappedTwitterResponse(response_typ, TwitterResponse):
        __doc__ = TwitterResponse.__doc__

    return WrappedTwitterResponse(response)

class TwitterCall(object):

    def __init__(
        self, auth, format, domain, callable_cls, uri="",
        uriparts=None, secure=True):
        self.auth = auth
        self.format = format
        self.domain = domain
        self.callable_cls = callable_cls
        self.uri = uri
        self.uriparts = uriparts
        self.secure = secure

    def __getattr__(self, k):
        try:
            return object.__getattr__(self, k)
        except AttributeError:
            def extend_call(arg):
                return self.callable_cls(
                    auth=self.auth, format=self.format, domain=self.domain,
                    callable_cls=self.callable_cls, uriparts=self.uriparts \
                        + (arg,),
                    secure=self.secure)
            if k == "_":
                return extend_call
            else:
                return extend_call(k)

    def __call__(self, **kwargs):
        # Build the uri.
        uriparts = []
        for uripart in self.uriparts:
            # If this part matches a keyword argument, use the
            # supplied value otherwise, just use the part.
            uriparts.append(str(kwargs.pop(uripart, uripart)))
        uri = '/'.join(uriparts)

        method = "GET"
        for action in POST_ACTIONS:
            if uri.endswith(action):
                method = "POST"
                break

        # If an id kwarg is present and there is no id to fill in in
        # the list of uriparts, assume the id goes at the end.
        id = kwargs.pop('id', None)
        if id:
            uri += "/%s" %(id)

        secure_str = ''
        if self.secure:
            secure_str = 's'
        dot = ""
        if self.format:
            dot = "."
        uriBase = "http%s://%s/%s%s%s" %(
                    secure_str, self.domain, uri, dot, self.format)

        headers = {}
        if self.auth:
            headers.update(self.auth.generate_headers())
            arg_data = self.auth.encode_params(uriBase, method, kwargs)
            if method == 'GET':
                uriBase += '?' + arg_data
                body = None
            else:
                body = arg_data.encode('utf8')

        req = urllib_request.Request(uriBase, body, headers)
        return self._handle_response(req, uri, arg_data)

    def _handle_response(self, req, uri, arg_data):
        try:
            handle = urllib_request.urlopen(req)
            if "json" == self.format:
                res = json.loads(handle.read().decode('utf8'))
                return wrap_response(res, handle.headers)
            else:
                return wrap_response(
                    handle.read().decode('utf8'), handle.headers)
        except urllib_error.HTTPError as e:
            if (e.code == 304):
                return []
            else:
                raise TwitterHTTPError(e, uri, self.format, arg_data)

class Twitter(TwitterCall):
    """
    The minimalist yet fully featured Twitter API class.

    Get RESTful data by accessing members of this class. The result
    is decoded python objects (lists and dicts).

    The Twitter API is documented here:

      http://dev.twitter.com/doc

    Examples::

      twitter = Twitter(
          auth=OAuth(token, token_key, con_secret, con_secret_key)))

      # Get the public timeline
      twitter.statuses.public_timeline()

      # Get a particular friend's timeline
      twitter.statuses.friends_timeline(id="billybob")

      # Also supported (but totally weird)
      twitter.statuses.friends_timeline.billybob()

      # Send a direct message
      twitter.direct_messages.new(
          user="billybob",
          text="I think yer swell!")

      # Get the members of a particular list of a particular friend
      twitter.user.listname.members(user="billybob", listname="billysbuds")

    Searching Twitter::

      twitter_search = Twitter(domain="search.twitter.com")

      # Find the latest search trends
      twitter_search.trends()

      # Search for the latest News on #gaza
      twitter_search.search(q="#gaza")

    Using the data returned
    -----------------------

    Twitter API calls return decoded JSON. This is converted into
    a bunch of Python lists, dicts, ints, and strings. For example::

      x = twitter.statuses.public_timeline()

      # The first 'tweet' in the timeline
      x[0]

      # The screen name of the user who wrote the first 'tweet'
      x[0]['user']['screen_name']

    Getting raw XML data
    --------------------

    If you prefer to get your Twitter data in XML format, pass
    format="xml" to the Twitter object when you instantiate it::

      twitter = Twitter(format="xml")

      The output will not be parsed in any way. It will be a raw string
      of XML.

    """
    def __init__(
        self, format="json",
        domain="api.twitter.com", secure=True, auth=None,
        api_version=_DEFAULT):
        """
        Create a new twitter API connector.

        Pass an `auth` parameter to use the credentials of a specific
        user. Generally you'll want to pass an `OAuth`
        instance::

            twitter = Twitter(auth=OAuth(
                    token, token_secret, consumer_key, consumer_secret))

        `domain` lets you change the domain you are connecting. By
        default it's `api.twitter.com` but `search.twitter.com` may be
        useful too.

        If `secure` is False you will connect with HTTP instead of
        HTTPS.

        `api_version` is used to set the base uri. By default it's
        '1'. If you are using "search.twitter.com" set this to None.
        """
        if not auth:
            auth = NoAuth()

        if (format not in ("json", "xml", "")):
            raise ValueError("Unknown data format '%s'" %(format))

        if api_version is _DEFAULT:
            if domain == 'api.twitter.com':
                api_version = '1'
            else:
                api_version = None

        uriparts = ()
        if api_version:
            uriparts += (str(api_version),)

        TwitterCall.__init__(
            self, auth=auth, format=format, domain=domain,
            callable_cls=TwitterCall,
            secure=secure, uriparts=uriparts)

__all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]
<|endoftext|>


<commit_before>import DefaultTable
import sstruct
from fontTools.misc.textTools import safeEval, num2binary, binary2num

# panose classification

panoseFormat = """
	bFamilyType:        B
	bSerifStyle:        B
	bWeight:            B
	bProportion:        B
	bContrast:          B
	bStrokeVariation:   B
	bArmStyle:          B
	bLetterForm:        B
	bMidline:           B
	bXHeight:           B
"""

class Panose:

	def toXML(self, writer, ttFont):
		formatstring, names, fixes = sstruct.getformat(panoseFormat)
		for name in names:
			writer.simpletag(name, value=getattr(self, name))
			writer.newline()

	def fromXML(self, (name, attrs, content), ttFont):
		setattr(self, name, safeEval(attrs["value"]))

# 'sfnt' OS/2 and Windows Metrics table - 'OS/2'

OS2_format_0 = """
	>   # big endian
	version:                H       # version
	xAvgCharWidth:          h       # average character width
	usWeightClass:          H       # degree of thickness of strokes
	usWidthClass:           H       # aspect ratio
	fsType:                 h       # type flags
	ySubscriptXSize:        h       # subscript horizontal font size
	ySubscriptYSize:        h       # subscript vertical font size
	ySubscriptXOffset:      h       # subscript x offset
	ySubscriptYOffset:      h       # subscript y offset
	ySuperscriptXSize:      h       # superscript horizontal font size
	ySuperscriptYSize:      h       # superscript vertical font size
	ySuperscriptXOffset:    h       # superscript x offset
	ySuperscriptYOffset:    h       # superscript y offset
	yStrikeoutSize:         h       # strikeout size
	yStrikeoutPosition:     h       # strikeout position
	sFamilyClass:           h       # font family class and subclass
	panose:                 10s     # panose classification number
	ulUnicodeRange1:        l       # character range
	ulUnicodeRange2:        l       # character range
	ulUnicodeRange3:        l       # character range
	ulUnicodeRange4:        l       # character range
	achVendID:              4s      # font vendor identification
	fsSelection:            H       # font selection flags
	fsFirstCharIndex:       H       # first unicode character index
	fsLastCharIndex:        H       # last unicode character index
	usTypoAscender:         H       # typographic ascender
	usTypoDescender:        H       # typographic descender
	usTypoLineGap:          H       # typographic line gap
	usWinAscent:            H       # Windows ascender
	usWinDescent:           H       # Windows descender
"""

OS2_format_1_addition =  """
	ulCodePageRange1:   l
	ulCodePageRange2:   l
"""

OS2_format_2_addition =  OS2_format_1_addition + """
	sxHeight:           h
	sCapHeight:         h
	usDefaultChar:      H
	usBreakChar:        H
	usMaxContex:        H
"""

bigendian = "	>	# big endian\n"

OS2_format_1 = OS2_format_0 + OS2_format_1_addition
OS2_format_2 = OS2_format_0 + OS2_format_2_addition
OS2_format_1_addition = bigendian + OS2_format_1_addition
OS2_format_2_addition = bigendian + OS2_format_2_addition

class table_O_S_2f_2(DefaultTable.DefaultTable):

	"""the OS/2 table"""

	def decompile(self, data, ttFont):
		dummy, data = sstruct.unpack2(OS2_format_0, data, self)
		if self.version == 1 and not data:
			# workaround for buggy Apple fonts
			self.version = 0
		if self.version == 1:
			sstruct.unpack(OS2_format_1_addition, data, self)
		elif self.version == 2:
			sstruct.unpack(OS2_format_2_addition, data, self)
		elif self.version <> 0:
			from fontTools import ttLib
			raise ttLib.TTLibError, "unknown format for OS/2 table: version %s" % self.version
		self.panose = sstruct.unpack(panoseFormat, self.panose, Panose())

	def compile(self, ttFont):
		panose = self.panose
		self.panose = sstruct.pack(panoseFormat, self.panose)
		if self.version == 0:
			data = sstruct.pack(OS2_format_0, self)
		elif self.version == 1:
			data = sstruct.pack(OS2_format_1, self)
		elif self.version == 2:
			data = sstruct.pack(OS2_format_2, self)
		else:
			from fontTools import ttLib
			raise ttLib.TTLibError, "unknown format for OS/2 table: version %s" % self.version
		self.panose = panose
		return data

	def toXML(self, writer, ttFont):
		if self.version == 1:
			format = OS2_format_1
		elif self.version == 2:
			format = OS2_format_2
		else:
			format = OS2_format_0
		formatstring, names, fixes = sstruct.getformat(format)
		for name in names:
			value = getattr(self, name)
			if type(value) == type(0L):
				value = int(value)
			if name=="panose":
				writer.begintag("panose")
				writer.newline()
				value.toXML(writer, ttFont)
				writer.endtag("panose")
			elif name in ("ulUnicodeRange1", "ulUnicodeRange2",
					"ulUnicodeRange3", "ulUnicodeRange4",
					"ulCodePageRange1", "ulCodePageRange2"):
				writer.simpletag(name, value=num2binary(value))
			elif name in ("fsType", "fsSelection"):
				writer.simpletag(name, value=num2binary(value, 16))
			elif name == "achVendID":
				writer.simpletag(name, value=repr(value)[1:-1])
			else:
				writer.simpletag(name, value=value)
			writer.newline()

	def fromXML(self, (name, attrs, content), ttFont):
		if name == "panose":
			self.panose = panose = Panose()
			for element in content:
				if type(element) == type(()):
					panose.fromXML(element, ttFont)
		elif name in ("ulUnicodeRange1", "ulUnicodeRange2",
				"ulUnicodeRange3", "ulUnicodeRange4",
				"ulCodePageRange1", "ulCodePageRange2",
				"fsType", "fsSelection"):
			setattr(self, name, binary2num(attrs["value"]))
		elif name == "achVendID":
			setattr(self, name, safeEval("'''" + attrs["value"] + "'''"))
		else:
			setattr(self, name, safeEval(attrs["value"]))

<commit_msg>spec changed: three fields are now unsigned instead of signed, and due to the the wonderful <snort> hungarian notation the field names changed as well... (So this change is not b/w compatible)<commit_after>import DefaultTable
import sstruct
from fontTools.misc.textTools import safeEval, num2binary, binary2num

# panose classification

panoseFormat = """
	bFamilyType:        B
	bSerifStyle:        B
	bWeight:            B
	bProportion:        B
	bContrast:          B
	bStrokeVariation:   B
	bArmStyle:          B
	bLetterForm:        B
	bMidline:           B
	bXHeight:           B
"""

class Panose:

	def toXML(self, writer, ttFont):
		formatstring, names, fixes = sstruct.getformat(panoseFormat)
		for name in names:
			writer.simpletag(name, value=getattr(self, name))
			writer.newline()

	def fromXML(self, (name, attrs, content), ttFont):
		setattr(self, name, safeEval(attrs["value"]))

# 'sfnt' OS/2 and Windows Metrics table - 'OS/2'

OS2_format_0 = """
	>   # big endian
	version:                H       # version
	xAvgCharWidth:          h       # average character width
	usWeightClass:          H       # degree of thickness of strokes
	usWidthClass:           H       # aspect ratio
	fsType:                 h       # type flags
	ySubscriptXSize:        h       # subscript horizontal font size
	ySubscriptYSize:        h       # subscript vertical font size
	ySubscriptXOffset:      h       # subscript x offset
	ySubscriptYOffset:      h       # subscript y offset
	ySuperscriptXSize:      h       # superscript horizontal font size
	ySuperscriptYSize:      h       # superscript vertical font size
	ySuperscriptXOffset:    h       # superscript x offset
	ySuperscriptYOffset:    h       # superscript y offset
	yStrikeoutSize:         h       # strikeout size
	yStrikeoutPosition:     h       # strikeout position
	sFamilyClass:           h       # font family class and subclass
	panose:                 10s     # panose classification number
	ulUnicodeRange1:        l       # character range
	ulUnicodeRange2:        l       # character range
	ulUnicodeRange3:        l       # character range
	ulUnicodeRange4:        l       # character range
	achVendID:              4s      # font vendor identification
	fsSelection:            H       # font selection flags
	fsFirstCharIndex:       H       # first unicode character index
	fsLastCharIndex:        H       # last unicode character index
	sTypoAscender:          h       # typographic ascender
	sTypoDescender:         h       # typographic descender
	sTypoLineGap:           h       # typographic line gap
	usWinAscent:            H       # Windows ascender
	usWinDescent:           H       # Windows descender
"""

OS2_format_1_addition =  """
	ulCodePageRange1:   l
	ulCodePageRange2:   l
"""

OS2_format_2_addition =  OS2_format_1_addition + """
	sxHeight:           h
	sCapHeight:         h
	usDefaultChar:      H
	usBreakChar:        H
	usMaxContex:        H
"""

bigendian = "	>	# big endian\n"

OS2_format_1 = OS2_format_0 + OS2_format_1_addition
OS2_format_2 = OS2_format_0 + OS2_format_2_addition
OS2_format_1_addition = bigendian + OS2_format_1_addition
OS2_format_2_addition = bigendian + OS2_format_2_addition

class table_O_S_2f_2(DefaultTable.DefaultTable):

	"""the OS/2 table"""

	def decompile(self, data, ttFont):
		dummy, data = sstruct.unpack2(OS2_format_0, data, self)
		if self.version == 1 and not data:
			# workaround for buggy Apple fonts
			self.version = 0
		if self.version == 1:
			sstruct.unpack(OS2_format_1_addition, data, self)
		elif self.version == 2:
			sstruct.unpack(OS2_format_2_addition, data, self)
		elif self.version <> 0:
			from fontTools import ttLib
			raise ttLib.TTLibError, "unknown format for OS/2 table: version %s" % self.version
		self.panose = sstruct.unpack(panoseFormat, self.panose, Panose())

	def compile(self, ttFont):
		panose = self.panose
		self.panose = sstruct.pack(panoseFormat, self.panose)
		if self.version == 0:
			data = sstruct.pack(OS2_format_0, self)
		elif self.version == 1:
			data = sstruct.pack(OS2_format_1, self)
		elif self.version == 2:
			data = sstruct.pack(OS2_format_2, self)
		else:
			from fontTools import ttLib
			raise ttLib.TTLibError, "unknown format for OS/2 table: version %s" % self.version
		self.panose = panose
		return data

	def toXML(self, writer, ttFont):
		if self.version == 1:
			format = OS2_format_1
		elif self.version == 2:
			format = OS2_format_2
		else:
			format = OS2_format_0
		formatstring, names, fixes = sstruct.getformat(format)
		for name in names:
			value = getattr(self, name)
			if type(value) == type(0L):
				value = int(value)
			if name=="panose":
				writer.begintag("panose")
				writer.newline()
				value.toXML(writer, ttFont)
				writer.endtag("panose")
			elif name in ("ulUnicodeRange1", "ulUnicodeRange2",
					"ulUnicodeRange3", "ulUnicodeRange4",
					"ulCodePageRange1", "ulCodePageRange2"):
				writer.simpletag(name, value=num2binary(value))
			elif name in ("fsType", "fsSelection"):
				writer.simpletag(name, value=num2binary(value, 16))
			elif name == "achVendID":
				writer.simpletag(name, value=repr(value)[1:-1])
			else:
				writer.simpletag(name, value=value)
			writer.newline()

	def fromXML(self, (name, attrs, content), ttFont):
		if name == "panose":
			self.panose = panose = Panose()
			for element in content:
				if type(element) == type(()):
					panose.fromXML(element, ttFont)
		elif name in ("ulUnicodeRange1", "ulUnicodeRange2",
				"ulUnicodeRange3", "ulUnicodeRange4",
				"ulCodePageRange1", "ulCodePageRange2",
				"fsType", "fsSelection"):
			setattr(self, name, binary2num(attrs["value"]))
		elif name == "achVendID":
			setattr(self, name, safeEval("'''" + attrs["value"] + "'''"))
		else:
			setattr(self, name, safeEval(attrs["value"]))

<|endoftext|>


<commit_before># Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
from distutils.version import LooseVersion

logger = logging.getLogger(__name__)

REQUIREMENTS = {
    "syutil>=0.0.5": ["syutil"],
    "Twisted==14.0.2": ["twisted==14.0.2"],
    "service_identity>=1.0.0": ["service_identity>=1.0.0"],
    "pyopenssl>=0.14": ["OpenSSL>=0.14"],
    "pyyaml": ["yaml"],
    "pyasn1": ["pyasn1"],
    "pynacl": ["nacl"],
    "daemonize": ["daemonize"],
    "py-bcrypt": ["bcrypt"],
    "frozendict>=0.4": ["frozendict"],
    "pillow": ["PIL"],
    "pydenticon": ["pydenticon"],
}
CONDITIONAL_REQUIREMENTS = {
    "web_client": {
        "matrix_angular_sdk>=0.6.5": ["syweb>=0.6.5"],
    }
}

def requirements(config=None, include_conditional=False):
    reqs = REQUIREMENTS.copy()
    for key, req in CONDITIONAL_REQUIREMENTS.items():
        if (config and getattr(config, key)) or include_conditional:
            reqs.update(req)
    return reqs

def github_link(project, version, egg):
    return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)

DEPENDENCY_LINKS = [
    github_link(
        project="pyca/pynacl",
        version="d4d3175589b892f6ea7c22f466e0e223853516fa",
        egg="pynacl-0.3.0",
    ),
    github_link(
        project="matrix-org/syutil",
        version="v0.0.5",
        egg="syutil-0.0.5",
    ),
    github_link(
        project="matrix-org/matrix-angular-sdk",
        version="v0.6.5",
        egg="matrix_angular_sdk-0.6.5",
    ),
]

class MissingRequirementError(Exception):
    pass

def check_requirements(config=None):
    """Checks that all the modules needed by synapse have been correctly
    installed and are at the correct version"""
    for dependency, module_requirements in (
            requirements(config, include_conditional=False).items()):
        for module_requirement in module_requirements:
            if ">=" in module_requirement:
                module_name, required_version = module_requirement.split(">=")
                version_test = ">="
            elif "==" in module_requirement:
                module_name, required_version = module_requirement.split("==")
                version_test = "=="
            else:
                module_name = module_requirement
                version_test = None

            try:
                module = __import__(module_name)
            except ImportError:
                logging.exception(
                    "Can't import %r which is part of %r",
                    module_name, dependency
                )
                raise MissingRequirementError(
                    "Can't import %r which is part of %r"
                    % (module_name, dependency)
                )
            version = getattr(module, "__version__", None)
            file_path = getattr(module, "__file__", None)
            logger.info(
                "Using %r version %r from %r to satisfy %r",
                module_name, version, file_path, dependency
            )

            if version_test == ">=":
                if version is None:
                    raise MissingRequirementError(
                        "Version of %r isn't set as __version__ of module %r"
                        % (dependency, module_name)
                    )
                if LooseVersion(version) < LooseVersion(required_version):
                    raise MissingRequirementError(
                        "Version of %r in %r is too old. %r < %r"
                        % (dependency, file_path, version, required_version)
                    )
            elif version_test == "==":
                if version is None:
                    raise MissingRequirementError(
                        "Version of %r isn't set as __version__ of module %r"
                        % (dependency, module_name)
                    )
                if LooseVersion(version) != LooseVersion(required_version):
                    raise MissingRequirementError(
                        "Unexpected version of %r in %r. %r != %r"
                        % (dependency, file_path, version, required_version)
                    )

def list_requirements():
    result = []
    linked = []
    for link in DEPENDENCY_LINKS:
        egg = link.split("#egg=")[1]
        linked.append(egg.split('-')[0])
        result.append(link)
    for requirement in requirements(include_conditional=True):
        is_linked = False
        for link in linked:
            if requirement.replace('-', '_').startswith(link):
                is_linked = True
        if not is_linked:
            result.append(requirement)
    return result

if __name__ == "__main__":
    import sys
    sys.stdout.writelines(req + "\n" for req in list_requirements())
<commit_msg>bump syutil to 0.0.6<commit_after># Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
from distutils.version import LooseVersion

logger = logging.getLogger(__name__)

REQUIREMENTS = {
    "syutil>=0.0.6": ["syutil>=0.0.6"],
    "Twisted==14.0.2": ["twisted==14.0.2"],
    "service_identity>=1.0.0": ["service_identity>=1.0.0"],
    "pyopenssl>=0.14": ["OpenSSL>=0.14"],
    "pyyaml": ["yaml"],
    "pyasn1": ["pyasn1"],
    "pynacl": ["nacl"],
    "daemonize": ["daemonize"],
    "py-bcrypt": ["bcrypt"],
    "frozendict>=0.4": ["frozendict"],
    "pillow": ["PIL"],
    "pydenticon": ["pydenticon"],
}
CONDITIONAL_REQUIREMENTS = {
    "web_client": {
        "matrix_angular_sdk>=0.6.5": ["syweb>=0.6.5"],
    }
}

def requirements(config=None, include_conditional=False):
    reqs = REQUIREMENTS.copy()
    for key, req in CONDITIONAL_REQUIREMENTS.items():
        if (config and getattr(config, key)) or include_conditional:
            reqs.update(req)
    return reqs

def github_link(project, version, egg):
    return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)

DEPENDENCY_LINKS = [
    github_link(
        project="pyca/pynacl",
        version="d4d3175589b892f6ea7c22f466e0e223853516fa",
        egg="pynacl-0.3.0",
    ),
    github_link(
        project="matrix-org/syutil",
        version="v0.0.6",
        egg="syutil-0.0.6",
    ),
    github_link(
        project="matrix-org/matrix-angular-sdk",
        version="v0.6.5",
        egg="matrix_angular_sdk-0.6.5",
    ),
]

class MissingRequirementError(Exception):
    pass

def check_requirements(config=None):
    """Checks that all the modules needed by synapse have been correctly
    installed and are at the correct version"""
    for dependency, module_requirements in (
            requirements(config, include_conditional=False).items()):
        for module_requirement in module_requirements:
            if ">=" in module_requirement:
                module_name, required_version = module_requirement.split(">=")
                version_test = ">="
            elif "==" in module_requirement:
                module_name, required_version = module_requirement.split("==")
                version_test = "=="
            else:
                module_name = module_requirement
                version_test = None

            try:
                module = __import__(module_name)
            except ImportError:
                logging.exception(
                    "Can't import %r which is part of %r",
                    module_name, dependency
                )
                raise MissingRequirementError(
                    "Can't import %r which is part of %r"
                    % (module_name, dependency)
                )
            version = getattr(module, "__version__", None)
            file_path = getattr(module, "__file__", None)
            logger.info(
                "Using %r version %r from %r to satisfy %r",
                module_name, version, file_path, dependency
            )

            if version_test == ">=":
                if version is None:
                    raise MissingRequirementError(
                        "Version of %r isn't set as __version__ of module %r"
                        % (dependency, module_name)
                    )
                if LooseVersion(version) < LooseVersion(required_version):
                    raise MissingRequirementError(
                        "Version of %r in %r is too old. %r < %r"
                        % (dependency, file_path, version, required_version)
                    )
            elif version_test == "==":
                if version is None:
                    raise MissingRequirementError(
                        "Version of %r isn't set as __version__ of module %r"
                        % (dependency, module_name)
                    )
                if LooseVersion(version) != LooseVersion(required_version):
                    raise MissingRequirementError(
                        "Unexpected version of %r in %r. %r != %r"
                        % (dependency, file_path, version, required_version)
                    )

def list_requirements():
    result = []
    linked = []
    for link in DEPENDENCY_LINKS:
        egg = link.split("#egg=")[1]
        linked.append(egg.split('-')[0])
        result.append(link)
    for requirement in requirements(include_conditional=True):
        is_linked = False
        for link in linked:
            if requirement.replace('-', '_').startswith(link):
                is_linked = True
        if not is_linked:
            result.append(requirement)
    return result

if __name__ == "__main__":
    import sys
    sys.stdout.writelines(req + "\n" for req in list_requirements())
<|endoftext|>


<commit_before>"""Utility sub-module for mir-eval"""

import numpy as np

def f_measure(precision, recall, beta=1.0):
    '''Compute the f-measure from precision and recall scores.

    :parameters:
        - precision : float in (0, 1]
            Precision

        - recall : float in (0, 1]
            Recall

        - beta : float > 0
            Weighting factor for f-measure

    :returns:
        - f_measure : float
            The weighted f-measure
    '''

    if precision == 0 and recall == 0:
        return 0.0

    return (1 + beta**2) * precision * recall / ((beta**2) * precision + recall)

def segments_to_boundaries(times, labels=None, label_prefix='__'):
    '''Convert segment start-end times into boundaries.
    '''

    boundaries = np.unique(np.ravel(times))

    if labels is None:
        boundary_labels = None
    else:
        boundary_labels = [seg_label for seg_label in labels]
        boundary_labels.append('%sEND' % label_prefix)

    return boundaries, boundary_labels

def boundaries_to_segments(boundaries, labels=None):
    '''Convert event boundaries into segments'''

    segments = np.asarray(zip(boundaries[:-1], boundaries[1:]))

    if labels is None:
        segment_labels = None
    else:
        segment_labels = labels[:-1]

    return segments, segment_labels

# TODO:   2014-01-17 15:45:13 by Brian McFee <brm2132@columbia.edu>
# boundaries => times, could be beat events
def adjust_boundaries(boundaries, labels=None, t_min=0.0, t_max=None, label_prefix='__'):
    '''Adjust the given list of boundary times to span the range [t_min, t_max].

    Any boundaries outside of the specified range will be removed.

    If the boundaries do not span [t_min, t_max], additional boundaries will be added.

    :parameters:
        - boundaries : np.array
            Array of boundary times (seconds)

        - labels : list or None
            Array of labels

        - t_min : float or None
            Minimum valid boundary time.

        - t_max : float or None
            Maximum valid boundary time.

        - label_prefix : str
            Prefix string to use for synthetic labels

    :returns:
        - new_boundaries : np.array
            Boundary times corrected to the given range.
    '''
    if t_min is not None:
        first_idx = np.argwhere(boundaries >= t_min)

        if len(first_idx) > 0:
            # We have boundaries below t_min
            # Crop them out
            if labels is not None:
                labels = labels[first_idx[0]:]
            boundaries = boundaries[first_idx[0]:]

        if boundaries[0] > t_min:
            # Lowest boundary is higher than t_min: add a new boundary and label
            boundaries = np.concatenate( ([t_min], boundaries) )
            if labels is not None:
                labels.insert(0, '%sT_MIN' % label_prefix)

    if t_max is not None:
        last_idx = np.argwhere(boundaries > t_max)

        if len(last_idx) > 0:
            # We have boundaries above t_max.
            # Trim to only boundaries <= t_max
            if labels is not None:
                labels = labels[:last_idx[0]]
            boundaries = boundaries[:last_idx[0]]

        if boundaries[-1] < t_max:
            # Last boundary is below t_max: add a new boundary and label
            boundaries = np.concatenate( (boundaries, [t_max]))
            if labels is not None:
                labels.append('%sT_MAX' % label_prefix)

    return boundaries, labels

def nextpow2(x):
    '''Compute the smallest n such that 2^n >= x
    '''
    return np.ceil(np.log2(x))

def fftfilt(b, x, n_fft=None):
    '''Raw translation of MATLAB fftfilt function (for only 1-d signal).
        Filter the 1-d signal x with the FIR filter described by the
        coefficients in b using the overlap-add method. If the FFT
        length n_fft is not specified, it and the overlap-add block length
        are selected so as to minimize the computational cost of
        the filtering operation.

    :parameters:
      - b : np.array
          FIR filter coefficients

      - x : np.array
          Input signal to be filtered

      - n_fft : float or None
          FFT size

    :returns:
      - y : np.array
          The filtered output signal

    :raises:
      - ValueError
          If the FIR filter is too long (> 2^20)
    '''
    nx = len(x)
    nb = len(b)
    if nb > 2**20:
        raise ValueError('Filters of length greater than 2^20 are not supported.')

    # Determine the FFT and block lengths to use:
    if n_fft is None:
        if nb >= nx:
            n_fft = 2**nextpow2(nb + nx - 1)
            L = nx
        else:
            fft_flops = np.array([18, 59, 138, 303, 660, 1441, 3150, 6875,
                                  14952, 32373, 69762, 149647, 319644, 680105,
                                  1441974, 3047619, 6422736, 13500637,
                                  28311786, 59244791])
            n = 2**np.arange(1, 21)
            dum = (n > (nb - 1))
            n, fft_flops = n[dum], fft_flops[dum]
            L = n - nb + 1
            idx = np.argmin(np.ceil(float(nx) / L) * fft_flops)
            n_fft, L = n[idx], L[idx]
    else:
        if n_fft < nb:
            n_fft = nb
        n_fft = 2**nextpow2(n_fft)
        L = n_fft - nb + 1

    n_fft = int(n_fft)
    B = np.fft.fft(b, n=n_fft)
    y = np.zeros_like(x, dtype='complex')

    # overlap and add
    istart = 0
    while istart < nx:
        iend = min(istart + L, nx)
        if iend == istart + 1:
            # need to fft a scalar
            X = x[istart] * np.ones((n_fft, 1))
        else:
            X = np.fft.fft(x[istart: iend], n=n_fft)
        Y = np.fft.ifft(X * B)
        yend = min(nx, istart + n_fft)
        y[istart: yend] += Y[:(yend - istart)]
        istart += L
    if not (np.any(np.imag(x)) or np.any(np.imag(b))):
        y = np.real(y)
    return y
<commit_msg>cleanup in adjust times<commit_after>"""Utility sub-module for mir-eval"""

import numpy as np

def f_measure(precision, recall, beta=1.0):
    '''Compute the f-measure from precision and recall scores.

    :parameters:
        - precision : float in (0, 1]
            Precision

        - recall : float in (0, 1]
            Recall

        - beta : float > 0
            Weighting factor for f-measure

    :returns:
        - f_measure : float
            The weighted f-measure
    '''

    if precision == 0 and recall == 0:
        return 0.0

    return (1 + beta**2) * precision * recall / ((beta**2) * precision + recall)

def segments_to_boundaries(times, labels=None, label_prefix='__'):
    '''Convert segment start-end times into boundaries.
    '''

    boundaries = np.unique(np.ravel(times))

    if labels is None:
        boundary_labels = None
    else:
        boundary_labels = [seg_label for seg_label in labels]
        boundary_labels.append('%sEND' % label_prefix)

    return boundaries, boundary_labels

def boundaries_to_segments(boundaries, labels=None):
    '''Convert event boundaries into segments'''

    segments = np.asarray(zip(boundaries[:-1], boundaries[1:]))

    if labels is None:
        segment_labels = None
    else:
        segment_labels = labels[:-1]

    return segments, segment_labels

def adjust_times(times, labels=None, t_min=0.0, t_max=None, label_prefix='__'):
    '''Adjust the given list of event times to span the range [t_min, t_max].

    Any event times outside of the specified range will be removed.

    If the times do not span [t_min, t_max], additional events will be inserted.

    :parameters:
        - times : np.array
            Array of event times (seconds)

        - labels : list or None
            Array of labels

        - t_min : float or None
            Minimum valid event time.

        - t_max : float or None
            Maximum valid event time.

        - label_prefix : str
            Prefix string to use for synthetic labels

    :returns:
        - new_times : np.array
            Event times corrected to the given range.
    '''
    if t_min is not None:
        first_idx = np.argwhere(times>= t_min)

        if len(first_idx) > 0:
            # We have events below t_min
            # Crop them out
            if labels is not None:
                labels = labels[first_idx[0]:]
            times = times[first_idx[0]:]

        if times[0] > t_min:
            # Lowest boundary is higher than t_min: add a new boundary and label
            times = np.concatenate( ([t_min], times) )
            if labels is not None:
                labels.insert(0, '%sT_MIN' % label_prefix)

    if t_max is not None:
        last_idx = np.argwhere(times > t_max)

        if len(last_idx) > 0:
            # We have boundaries above t_max.
            # Trim to only boundaries <= t_max
            if labels is not None:
                labels = labels[:last_idx[0]]
            times = times[:last_idx[0]]

        if times[-1] < t_max:
            # Last boundary is below t_max: add a new boundary and label
            times = np.concatenate( (times, [t_max]))
            if labels is not None:
                labels.append('%sT_MAX' % label_prefix)

    return times, labels

def nextpow2(x):
    '''Compute the smallest n such that 2^n >= x
    '''
    return np.ceil(np.log2(x))

def fftfilt(b, x, n_fft=None):
    '''Raw translation of MATLAB fftfilt function (for only 1-d signal).
        Filter the 1-d signal x with the FIR filter described by the
        coefficients in b using the overlap-add method. If the FFT
        length n_fft is not specified, it and the overlap-add block length
        are selected so as to minimize the computational cost of
        the filtering operation.

    :parameters:
      - b : np.array
          FIR filter coefficients

      - x : np.array
          Input signal to be filtered

      - n_fft : float or None
          FFT size

    :returns:
      - y : np.array
          The filtered output signal

    :raises:
      - ValueError
          If the FIR filter is too long (> 2^20)
    '''
    nx = len(x)
    nb = len(b)
    if nb > 2**20:
        raise ValueError('Filters of length greater than 2^20 are not supported.')

    # Determine the FFT and block lengths to use:
    if n_fft is None:
        if nb >= nx:
            n_fft = 2**nextpow2(nb + nx - 1)
            L = nx
        else:
            fft_flops = np.array([18, 59, 138, 303, 660, 1441, 3150, 6875,
                                  14952, 32373, 69762, 149647, 319644, 680105,
                                  1441974, 3047619, 6422736, 13500637,
                                  28311786, 59244791])
            n = 2**np.arange(1, 21)
            dum = (n > (nb - 1))
            n, fft_flops = n[dum], fft_flops[dum]
            L = n - nb + 1
            idx = np.argmin(np.ceil(float(nx) / L) * fft_flops)
            n_fft, L = n[idx], L[idx]
    else:
        if n_fft < nb:
            n_fft = nb
        n_fft = 2**nextpow2(n_fft)
        L = n_fft - nb + 1

    n_fft = int(n_fft)
    B = np.fft.fft(b, n=n_fft)
    y = np.zeros_like(x, dtype='complex')

    # overlap and add
    istart = 0
    while istart < nx:
        iend = min(istart + L, nx)
        if iend == istart + 1:
            # need to fft a scalar
            X = x[istart] * np.ones((n_fft, 1))
        else:
            X = np.fft.fft(x[istart: iend], n=n_fft)
        Y = np.fft.ifft(X * B)
        yend = min(nx, istart + n_fft)
        y[istart: yend] += Y[:(yend - istart)]
        istart += L
    if not (np.any(np.imag(x)) or np.any(np.imag(b))):
        y = np.real(y)
    return y
<|endoftext|>


<commit_before>#!/usr/bin/env python

from argparse import ArgumentParser
from base64 import b64encode
from cStringIO import StringIO
from collections import OrderedDict
from os.path import basename
import json
import logging

from arrow import utcnow
from requests import Request, Session
from requests_foauth import Foauth
import dateutil.parser

logging.basicConfig()

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

parser = ArgumentParser(
    description='Delete your last 3200 tweets and backup tweeted photos to Google Drive',
    epilog='''Twittercide uses foauth.org <http://foauth.org/> to authenticate with Twitter and Google's APIs.

    In order to use it, you'll need to sign up with foauth.org, and authorize both those services <https://foauth.org/services/>.

    For Twitter, you need to check the option to "read and send tweets". For Google, you need "access your documents".
    '''
)
parser.add_argument('email', help='foauth.org email')
parser.add_argument('password', help='foauth.org password')
parser.add_argument('--days_ago', '-d', default=0, help="only delete tweets older than DAYS_AGO days. (A value of 0 will delete all tweets)", type=int)
parser.add_argument('--dry-run', action='store_true', help="don't delete any tweets, but backup tweeted photos")
parser.add_argument('--debug', '-v', action='store_true', help='show debug logging')

args = parser.parse_args()

if args.debug:
    log.setLevel(logging.DEBUG)

class Twittercider(object):

    def __init__(self, args):
        self.days_ago = args.days_ago
        self.dry_run = args.dry_run

        self.session = Session()
        self.session.mount('https://', Foauth(args.email, args.password))

        self.now = utcnow()

    def _get_or_create_parent_dir(self):
        metadata = {
            'title': 'Twittercide',
            'mimeType': 'application/vnd.google-apps.folder',
        }
        parent_dir = self._get_or_upload(metadata)

        self.parent_dir_id = parent_dir['id']

    def _prepare_upload(self, url, files):
        request = Request('post', url, files=files)
        request = self.session.prepare_request(request)

        # This is a bit ugly (ideally requests would expose a way to set the
        # boundary explictly), but it is the simplest way that doesn't require any
        # external libraries.
        request.headers['Content-Type'] = request.headers['Content-Type'].replace('form-data', 'related')

        return request

    def _prepare_query(self, metadata):
        query = list()

        for key, value in metadata.iteritems():
            if key == 'parents':
                for subvalue in value:
                    # When in the metadata, each parent item is object. To search, we just need the ID
                    subvalue = subvalue['id']

                    query.append('"{}" in {}'.format(subvalue, key))
            else:
                query.append('{} = "{}"'.format(key, value))

        query = ' and '.join(query)

        return query

    def _get_or_upload(self, metadata, file_=None):
        query = self._prepare_query(metadata)
        log.debug('Query {}'.format(query))

        response = self.session.get('https://www.googleapis.com/drive/v2/files', params={
            'q': query,
        })

        log.debug('Query response {}'.format(response.text))

        response.raise_for_status()

        response_data = response.json()
        results = response_data['items']

        try:
            result = results[0]
        except IndexError:
            log.debug('File not found in Google Drive. Uploading')

            # Google Drive expects multipart/related, and for the fields to be in a
            # particular order. First, the metadata, then the file.
            #
            # It expects the metadata as application/json, and the file as base64 (with
            # Content-Transfer-Encoding: base64).

            url = 'https://www.googleapis.com/upload/drive/v2/files'

            files = OrderedDict()
            files['metadata'] = ('metadata', json.dumps(metadata), 'application/json')

            if file_:
                url = 'https://www.googleapis.com/upload/drive/v2/files?uploadType=multipart'
                files['file'] = ('file', b64encode(file_.read()), 'image/png', {'Content-Transfer-Encoding': 'base64'})

            request = self._prepare_upload(url, files)
            response = self.session.send(request)

            log.debug('Upload response {}'.format(response.text))

            response.raise_for_status()

            response_data = response.json()
            result = response_data

        return result

    def _backup_twitter_media(self, url):
        title = basename(url)
        metadata = {
            'title': title,
            'parents': ({
                'kind': 'drive#file',
                'id': self.parent_dir_id,
            },)
        }

        file_response = self.session.get(url)
        file_response.raise_for_status()

        file_ = file_response.content
        file_ = StringIO(file_)

        self._get_or_upload(metadata, file_)

        log.info('Backed up {}'.format(url))

    def _backup_tweets(self):
        response = self.session.get('https://api.twitter.com/1.1/statuses/user_timeline.json', params={
            'count': 200,
            'include_rts': False,
        })
        response.raise_for_status()

        results = response.json()

        for tweet in results:
            created_at = tweet['created_at']
            created_at = dateutil.parser.parse(created_at)

            delta = self.now - created_at

            if delta.days >= self.days_ago:
                if 'extended_entities' in tweet:
                    if 'media' in tweet['extended_entities']:
                        for media in tweet['extended_entities']['media']:
                            self._backup_twitter_media(media['media_url'])

                if not self.dry_run:
                    log.info('Pretending to delete {}'.format(tweet['id_str']))

                    # self.post('https://api.twitter.com/1.1/statuses/destroy/{}.json'.format(tweet['id_str'])
            else:
                log.debug('Tweet not old enough ({} days old, must be at least {}). Skipping'.format(delta.days, self.days_ago))

        max_id = tweet['id_str']

    def twittercide(self):
        self._get_or_create_parent_dir()
        self._backup_tweets()

if __name__ == '__main__':
    t = Twittercider(args)
    t.twittercide()
<commit_msg>Fetch all tweets<commit_after>#!/usr/bin/env python

from argparse import ArgumentParser
from base64 import b64encode
from cStringIO import StringIO
from collections import OrderedDict
from os.path import basename
import json
import logging

from arrow import utcnow
from requests import Request, Session
from requests_foauth import Foauth
import dateutil.parser

logging.basicConfig()

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

parser = ArgumentParser(
    description='Delete your last 3200 tweets and backup tweeted photos to Google Drive',
    epilog='''Twittercide uses foauth.org <http://foauth.org/> to authenticate with Twitter and Google's APIs.

    In order to use it, you'll need to sign up with foauth.org, and authorize both those services <https://foauth.org/services/>.

    For Twitter, you need to check the option to "read and send tweets". For Google, you need "access your documents".
    '''
)
parser.add_argument('email', help='foauth.org email')
parser.add_argument('password', help='foauth.org password')
parser.add_argument('--days_ago', '-d', default=0, help="only delete tweets older than DAYS_AGO days. (A value of 0 will delete all tweets)", type=int)
parser.add_argument('--dry-run', action='store_true', help="don't delete any tweets, but backup tweeted photos")
parser.add_argument('--debug', '-v', action='store_true', help='show debug logging')

args = parser.parse_args()

if args.debug:
    log.setLevel(logging.DEBUG)

class Twittercider(object):

    def __init__(self, args):
        self.days_ago = args.days_ago
        self.dry_run = args.dry_run

        self.session = Session()
        self.session.mount('https://', Foauth(args.email, args.password))

        self.now = utcnow()

    def _get_or_create_parent_dir(self):
        metadata = {
            'title': 'Twittercide',
            'mimeType': 'application/vnd.google-apps.folder',
        }
        parent_dir = self._get_or_upload(metadata)

        self.parent_dir_id = parent_dir['id']

    def _prepare_upload(self, url, files):
        request = Request('post', url, files=files)
        request = self.session.prepare_request(request)

        # This is a bit ugly (ideally requests would expose a way to set the
        # boundary explictly), but it is the simplest way that doesn't require any
        # external libraries.
        request.headers['Content-Type'] = request.headers['Content-Type'].replace('form-data', 'related')

        return request

    def _prepare_query(self, metadata):
        query = list()

        for key, value in metadata.iteritems():
            if key == 'parents':
                for subvalue in value:
                    # When in the metadata, each parent item is object. To search, we just need the ID
                    subvalue = subvalue['id']

                    query.append('"{}" in {}'.format(subvalue, key))
            else:
                query.append('{} = "{}"'.format(key, value))

        query = ' and '.join(query)

        return query

    def _get_or_upload(self, metadata, file_=None):
        query = self._prepare_query(metadata)
        log.debug('Query {}'.format(query))

        response = self.session.get('https://www.googleapis.com/drive/v2/files', params={
            'q': query,
        })

        log.debug('Query response {}'.format(response.text))

        response.raise_for_status()

        response_data = response.json()
        results = response_data['items']

        try:
            result = results[0]
        except IndexError:
            log.debug('File not found in Google Drive. Uploading')

            # Google Drive expects multipart/related, and for the fields to be in a
            # particular order. First, the metadata, then the file.
            #
            # It expects the metadata as application/json, and the file as base64 (with
            # Content-Transfer-Encoding: base64).

            url = 'https://www.googleapis.com/upload/drive/v2/files'

            files = OrderedDict()
            files['metadata'] = ('metadata', json.dumps(metadata), 'application/json')

            if file_:
                url = 'https://www.googleapis.com/upload/drive/v2/files?uploadType=multipart'

                # TODO: Determine mimetype
                files['file'] = ('file', b64encode(file_.read()), 'image/png', {'Content-Transfer-Encoding': 'base64'})

            request = self._prepare_upload(url, files)
            response = self.session.send(request)

            log.debug('Upload response {}'.format(response.text))

            response.raise_for_status()

            response_data = response.json()
            result = response_data

        return result

    def _backup_twitter_media(self, url):
        title = basename(url)
        metadata = {
            'title': title,
            'parents': ({
                'kind': 'drive#file',
                'id': self.parent_dir_id,
            },)
        }

        file_response = self.session.get(url)
        file_response.raise_for_status()

        file_ = file_response.content
        file_ = StringIO(file_)

        self._get_or_upload(metadata, file_)

        log.info('Backed up {}'.format(url))

    def _backup_tweets(self):
        finished = False

        max_id = None
        while not finished:
            response = self.session.get('https://api.twitter.com/1.1/statuses/user_timeline.json', params={
                'count': 200,
                'include_rts': False,
                'max_id': max_id,
            })
            response.raise_for_status()

            log.debug('Loaded next 200 tweets')

            results = response.json()

            for tweet in results:
                created_at = tweet['created_at']
                created_at = dateutil.parser.parse(created_at)

                delta = self.now - created_at

                if delta.days >= self.days_ago:
                    if 'extended_entities' in tweet:
                        if 'media' in tweet['extended_entities']:
                            for media in tweet['extended_entities']['media']:
                                self._backup_twitter_media(media['media_url'])

                    if not self.dry_run:
                        log.info('Pretending to delete {}'.format(tweet['id_str']))

                        # self.post('https://api.twitter.com/1.1/statuses/destroy/{}.json'.format(tweet['id_str'])
                else:
                    log.debug('{} not old enough ({} days old, must be at least {}). Skipping'.format(tweet['id_str'], delta.days, self.days_ago))

            old_max_id = max_id
            max_id = tweet['id_str']

            finished = max_id == old_max_id

    def twittercide(self):
        self._get_or_create_parent_dir()
        self._backup_tweets()

if __name__ == '__main__':
    t = Twittercider(args)
    t.twittercide()
<|endoftext|>