<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import os
import sys
import argparse
import base58

from .client import DownstreamClient
from downstream_farmer.version import __version__
from downstream_farmer.exc import ConnectError, DownstreamError

try:
    from urllib2 import urlopen, URLError
except ImportError:
    from urllib.request import urlopen
    from urllib.error import URLError

def fail_exit(msg, exit_code=1):
    sys.stderr.write('Error: %s\n' % msg)
    sys.exit(exit_code)

def run_prototype(url, number):
    try:
        # generate a blank address...
        test_address = base58.b58encode_check(b'\x00'+os.urandom(20))
        client = DownstreamClient(test_address)

        print('Connect to server')
        client.connect(url)

        print('Fetching contract')
        client.get_chunk()

        print('Answering challenge 0')
        client.answer_challenge()

        i = 1
        while number == 0 or i < number:
            client.get_challenge()
            print('Answering challenge {0}'.format(i))
            client.answer_challenge()
            i += 1

        print('Verification successful!')

    except DownstreamError as e:
        fail_exit(e.message)

def check_connectivity(url):
        """ Check to see if we even get a connection to the server.
        https://stackoverflow.com/questions/3764291/checking-network-connection
        """
        try:
            urlopen(url, timeout=2)
        except URLError:
            raise ConnectError("Could not connect to server.")

def eval_args(args):
    try:
        check_connectivity(args.node_url)
    except ConnectError as e:
        fail_exit(e.message)

    if args.number < 0:
        fail_exit('Must specify a non-negative number of challenges.')

    run_prototype(args.node_url, args.number)

def parse_args():
    parser = argparse.ArgumentParser('downstream-farmer')
    parser.add_argument('-V', '--version', action='version',
                        version=__version__)
    parser.add_argument('node_url', help='URL of the Downstream node')
    parser.add_argument('-n', '--number', type=int,
                        default=0,
                        help='Number of challenges to perform.  '
                        'If unspecified, perform challenges until killed.')
    return parser.parse_args()

def main():
    args = parse_args()
    eval_args(args)
<commit_msg>added default for node_url to verify.driveshare.org<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import os
import sys
import argparse
import base58

from .client import DownstreamClient
from downstream_farmer.version import __version__
from downstream_farmer.exc import ConnectError, DownstreamError

try:
    from urllib2 import urlopen, URLError
except ImportError:
    from urllib.request import urlopen
    from urllib.error import URLError

def fail_exit(msg, exit_code=1):
    sys.stderr.write('Error: %s\n' % msg)
    sys.exit(exit_code)

def run_prototype(url, number):
    try:
        # generate a blank address...
        test_address = base58.b58encode_check(b'\x00'+os.urandom(20))
        client = DownstreamClient(test_address)

        print('Connect to server')
        client.connect(url)

        print('Fetching contract')
        client.get_chunk()

        print('Answering challenge 0')
        client.answer_challenge()

        i = 1
        while number == 0 or i < number:
            client.get_challenge()
            print('Answering challenge {0}'.format(i))
            client.answer_challenge()
            i += 1

        print('Verification successful!')

    except DownstreamError as e:
        fail_exit(e.message)

def check_connectivity(url):
        """ Check to see if we even get a connection to the server.
        https://stackoverflow.com/questions/3764291/checking-network-connection
        """
        try:
            urlopen(url, timeout=2)
        except URLError:
            raise ConnectError("Could not connect to server.")

def eval_args(args):
    try:
        check_connectivity(args.node_url)
    except ConnectError as e:
        fail_exit(e.message)

    if args.number < 0:
        fail_exit('Must specify a non-negative number of challenges.')

    run_prototype(args.node_url, args.number)

def parse_args():
    parser = argparse.ArgumentParser('downstream-farmer')
    parser.add_argument('-V', '--version', action='version',
                        version=__version__)
    parser.add_argument('node_url', help='URL of the Downstream node',
                        nargs='?',
                        default='http://verify.driveshare.org')
    parser.add_argument('-n', '--number', type=int,
                        default=0,
                        help='Number of challenges to perform. '
                        'If unspecified, perform challenges until killed.')
    return parser.parse_args()

def main():
    args = parse_args()
    eval_args(args)
<|endoftext|>


<commit_before>import base64
import binascii
import imghdr
import io
import uuid

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.contrib.postgres import fields as postgres_fields
from django.utils.translation import gettext_lazy as _
from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange
from rest_framework.fields import (
    DateField,
    DateTimeField,
    DictField,
    EmailField,
    FileField,
    FloatField,
    ImageField,
    IntegerField,
    DecimalField,
)
from rest_framework.serializers import ModelSerializer
from rest_framework.utils import html
from drf_extra_fields import compat

DEFAULT_CONTENT_TYPE = "application/octet-stream"

class Base64FieldMixin(object):
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError

    @property
    def INVALID_FILE_MESSAGE(self):
        raise NotImplementedError

    @property
    def INVALID_TYPE_MESSAGE(self):
        raise NotImplementedError

    EMPTY_VALUES = (None, '', [], (), {})

    def __init__(self, *args, **kwargs):
        self.represent_in_base64 = kwargs.pop('represent_in_base64', False)
        super(Base64FieldMixin, self).__init__(*args, **kwargs)

    def to_internal_value(self, base64_data):
        # Check if this is a base64 string
        if base64_data in self.EMPTY_VALUES:
            return None

        if isinstance(base64_data, str):
            # Strip base64 header.
            if ';base64,' in base64_data:
                header, base64_data = base64_data.split(';base64,')

            # Try to decode the file. Return validation error if it fails.
            try:
                decoded_file = base64.b64decode(base64_data)
            except (TypeError, binascii.Error, ValueError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)
            # Generate file name:
            file_name = self.get_file_name(decoded_file)
            # Get the file name extension:
            file_extension = self.get_file_extension(file_name, decoded_file)
            if file_extension not in self.ALLOWED_TYPES:
                raise ValidationError(self.INVALID_TYPE_MESSAGE)
            complete_file_name = file_name + "." + file_extension
            data = ContentFile(decoded_file, name=complete_file_name)
            return super(Base64FieldMixin, self).to_internal_value(data)
        raise ValidationError(_('Invalid type. This is not an base64 string: {}'.format(
            type(base64_data))))

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError

    def get_file_name(self, decoded_file):
        return str(uuid.uuid4())

    def to_representation(self, file):
        if self.represent_in_base64:
            # If the underlying ImageField is blank, a ValueError would be
            # raised on `open`. When representing as base64, simply return an
            # empty base64 str rather than let the exception propagate unhandled
            # up into serializers.
            if not file:
                return ''

            try:
                with open(file.path, 'rb') as f:
                    return base64.b64encode(f.read()).decode()
            except Exception:
                raise IOError("Error encoding file")
        else:
            return super(Base64FieldMixin, self).to_representation(file)

class Base64ImageField(Base64FieldMixin, ImageField):
    """
    A django-rest-framework field for handling image-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    ALLOWED_TYPES = (
        "jpeg",
        "jpg",
        "png",
        "gif"
    )
    INVALID_FILE_MESSAGE = _("Please upload a valid image.")
    INVALID_TYPE_MESSAGE = _("The type of the image couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        try:
            from PIL import Image
        except ImportError:
            raise ImportError("Pillow is not installed.")
        extension = imghdr.what(filename, decoded_file)

        # Try with PIL as fallback if format not detected due
        # to bug in imghdr https://bugs.python.org/issue16512
        if extension is None:
            try:
                image = Image.open(io.BytesIO(decoded_file))
            except (OSError, IOError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)

            extension = image.format.lower()

        extension = "jpg" if extension == "jpeg" else extension
        return extension

class HybridImageField(Base64ImageField):
    """
    A django-rest-framework field for handling image-uploads through
    raw post data, with a fallback to multipart form data.
    """

    def to_internal_value(self, data):
        """
        Try Base64Field first, and then try the ImageField
        ``to_internal_value``, MRO doesn't work here because
        Base64FieldMixin throws before ImageField can run.
        """
        try:
            return Base64FieldMixin.to_internal_value(self, data)
        except ValidationError:
            return ImageField.to_internal_value(self, data)

class Base64FileField(Base64FieldMixin, FileField):
    """
    A django-rest-framework field for handling file-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError('List allowed file extensions')

    INVALID_FILE_MESSAGE = _("Please upload a valid file.")
    INVALID_TYPE_MESSAGE = _("The type of the file couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError('Implement file validation and return matching extension.')

class RangeField(DictField):

    range_type = None

    default_error_messages = dict(DictField.default_error_messages)
    default_error_messages.update({
        'too_much_content': _('Extra content not allowed "{extra}".'),
        'bound_ordering': _('The start of the range must not exceed the end of the range.'),
    })

    def to_internal_value(self, data):
        """
        Range instances <- Dicts of primitive datatypes.
        """
        if html.is_html_input(data):
            data = html.parse_html_dict(data)
        if not isinstance(data, dict):
            self.fail('not_a_dict', input_type=type(data).__name__)

        # allow_empty is added to DictField in DRF Version 3.9.3
        if hasattr(self, "allow_empty") and not self.allow_empty and len(data) == 0:
            self.fail('empty')

        extra_content = list(set(data) - set(["lower", "upper", "bounds", "empty"]))
        if extra_content:
            self.fail('too_much_content', extra=', '.join(map(str, extra_content)))

        validated_dict = {}
        for key in ('lower', 'upper'):
            try:
                value = data[key]
            except KeyError:
                continue

            validated_dict[str(key)] = self.child.run_validation(value)

        lower, upper = validated_dict.get('lower'), validated_dict.get('upper')
        if lower is not None and upper is not None and lower > upper:
            self.fail('bound_ordering')

        for key in ('bounds', 'empty'):
            try:
                value = data[key]
            except KeyError:
                continue

            validated_dict[str(key)] = value

        return self.range_type(**validated_dict)

    def to_representation(self, value):
        """
        Range instances -> dicts of primitive datatypes.
        """
        if value.isempty:
            return {'empty': True}
        lower = self.child.to_representation(value.lower) if value.lower is not None else None
        upper = self.child.to_representation(value.upper) if value.upper is not None else None
        return {'lower': lower,
                'upper': upper,
                'bounds': value._bounds}

    def get_initial(self):
        initial = super().get_initial()
        return self.to_representation(initial)

class IntegerRangeField(RangeField):
    child = IntegerField()
    range_type = NumericRange

class FloatRangeField(RangeField):
    child = FloatField()
    range_type = NumericRange

class DecimalRangeField(RangeField):
    child = DecimalField(max_digits=None, decimal_places=None)
    range_type = NumericRange

class DateTimeRangeField(RangeField):
    child = DateTimeField()
    range_type = DateTimeTZRange

class DateRangeField(RangeField):
    child = DateField()
    range_type = DateRange

# monkey patch modelserializer to map Native django Range fields to
# drf_extra_fiels's Range fields.

ModelSerializer.serializer_field_mapping[postgres_fields.DateTimeRangeField] = DateTimeRangeField
ModelSerializer.serializer_field_mapping[postgres_fields.DateRangeField] = DateRangeField
ModelSerializer.serializer_field_mapping[postgres_fields.IntegerRangeField] = IntegerRangeField
ModelSerializer.serializer_field_mapping[postgres_fields.DecimalRangeField] = DecimalRangeField
if compat.FloatRangeField:
    ModelSerializer.serializer_field_mapping[compat.FloatRangeField] = FloatRangeField

class LowercaseEmailField(EmailField):
    """
    An enhancement over django-rest-framework's EmailField to allow
    case-insensitive serialization and deserialization of e-mail addresses.
    """
    def to_internal_value(self, data):
        data = super(LowercaseEmailField, self).to_internal_value(data)
        return data.lower()

    def to_representation(self, value):
        value = super(LowercaseEmailField, self).to_representation(value)
        return value.lower()
<commit_msg>make postgres dependency optional<commit_after>import base64
import binascii
import imghdr
import io
import uuid

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.utils.translation import gettext_lazy as _
from rest_framework.fields import (
    DateField,
    DateTimeField,
    DictField,
    EmailField,
    FileField,
    FloatField,
    ImageField,
    IntegerField,
    DecimalField,
)
from rest_framework.serializers import ModelSerializer
from rest_framework.utils import html
from drf_extra_fields import compat

try:
    from django.contrib.postgres import fields as postgres_fields
    from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange
except:
    postgres_fields = None

DEFAULT_CONTENT_TYPE = "application/octet-stream"

class Base64FieldMixin(object):
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError

    @property
    def INVALID_FILE_MESSAGE(self):
        raise NotImplementedError

    @property
    def INVALID_TYPE_MESSAGE(self):
        raise NotImplementedError

    EMPTY_VALUES = (None, '', [], (), {})

    def __init__(self, *args, **kwargs):
        self.represent_in_base64 = kwargs.pop('represent_in_base64', False)
        super(Base64FieldMixin, self).__init__(*args, **kwargs)

    def to_internal_value(self, base64_data):
        # Check if this is a base64 string
        if base64_data in self.EMPTY_VALUES:
            return None

        if isinstance(base64_data, str):
            # Strip base64 header.
            if ';base64,' in base64_data:
                header, base64_data = base64_data.split(';base64,')

            # Try to decode the file. Return validation error if it fails.
            try:
                decoded_file = base64.b64decode(base64_data)
            except (TypeError, binascii.Error, ValueError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)
            # Generate file name:
            file_name = self.get_file_name(decoded_file)
            # Get the file name extension:
            file_extension = self.get_file_extension(file_name, decoded_file)
            if file_extension not in self.ALLOWED_TYPES:
                raise ValidationError(self.INVALID_TYPE_MESSAGE)
            complete_file_name = file_name + "." + file_extension
            data = ContentFile(decoded_file, name=complete_file_name)
            return super(Base64FieldMixin, self).to_internal_value(data)
        raise ValidationError(_('Invalid type. This is not an base64 string: {}'.format(
            type(base64_data))))

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError

    def get_file_name(self, decoded_file):
        return str(uuid.uuid4())

    def to_representation(self, file):
        if self.represent_in_base64:
            # If the underlying ImageField is blank, a ValueError would be
            # raised on `open`. When representing as base64, simply return an
            # empty base64 str rather than let the exception propagate unhandled
            # up into serializers.
            if not file:
                return ''

            try:
                with open(file.path, 'rb') as f:
                    return base64.b64encode(f.read()).decode()
            except Exception:
                raise IOError("Error encoding file")
        else:
            return super(Base64FieldMixin, self).to_representation(file)

class Base64ImageField(Base64FieldMixin, ImageField):
    """
    A django-rest-framework field for handling image-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    ALLOWED_TYPES = (
        "jpeg",
        "jpg",
        "png",
        "gif"
    )
    INVALID_FILE_MESSAGE = _("Please upload a valid image.")
    INVALID_TYPE_MESSAGE = _("The type of the image couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        try:
            from PIL import Image
        except ImportError:
            raise ImportError("Pillow is not installed.")
        extension = imghdr.what(filename, decoded_file)

        # Try with PIL as fallback if format not detected due
        # to bug in imghdr https://bugs.python.org/issue16512
        if extension is None:
            try:
                image = Image.open(io.BytesIO(decoded_file))
            except (OSError, IOError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)

            extension = image.format.lower()

        extension = "jpg" if extension == "jpeg" else extension
        return extension

class HybridImageField(Base64ImageField):
    """
    A django-rest-framework field for handling image-uploads through
    raw post data, with a fallback to multipart form data.
    """

    def to_internal_value(self, data):
        """
        Try Base64Field first, and then try the ImageField
        ``to_internal_value``, MRO doesn't work here because
        Base64FieldMixin throws before ImageField can run.
        """
        try:
            return Base64FieldMixin.to_internal_value(self, data)
        except ValidationError:
            return ImageField.to_internal_value(self, data)

class Base64FileField(Base64FieldMixin, FileField):
    """
    A django-rest-framework field for handling file-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError('List allowed file extensions')

    INVALID_FILE_MESSAGE = _("Please upload a valid file.")
    INVALID_TYPE_MESSAGE = _("The type of the file couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError('Implement file validation and return matching extension.')

class RangeField(DictField):

    range_type = None

    default_error_messages = dict(DictField.default_error_messages)
    default_error_messages.update({
        'too_much_content': _('Extra content not allowed "{extra}".'),
        'bound_ordering': _('The start of the range must not exceed the end of the range.'),
    })

    def to_internal_value(self, data):
        """
        Range instances <- Dicts of primitive datatypes.
        """
        if html.is_html_input(data):
            data = html.parse_html_dict(data)
        if not isinstance(data, dict):
            self.fail('not_a_dict', input_type=type(data).__name__)

        # allow_empty is added to DictField in DRF Version 3.9.3
        if hasattr(self, "allow_empty") and not self.allow_empty and len(data) == 0:
            self.fail('empty')

        extra_content = list(set(data) - set(["lower", "upper", "bounds", "empty"]))
        if extra_content:
            self.fail('too_much_content', extra=', '.join(map(str, extra_content)))

        validated_dict = {}
        for key in ('lower', 'upper'):
            try:
                value = data[key]
            except KeyError:
                continue

            validated_dict[str(key)] = self.child.run_validation(value)

        lower, upper = validated_dict.get('lower'), validated_dict.get('upper')
        if lower is not None and upper is not None and lower > upper:
            self.fail('bound_ordering')

        for key in ('bounds', 'empty'):
            try:
                value = data[key]
            except KeyError:
                continue

            validated_dict[str(key)] = value

        return self.range_type(**validated_dict)

    def to_representation(self, value):
        """
        Range instances -> dicts of primitive datatypes.
        """
        if value.isempty:
            return {'empty': True}
        lower = self.child.to_representation(value.lower) if value.lower is not None else None
        upper = self.child.to_representation(value.upper) if value.upper is not None else None
        return {'lower': lower,
                'upper': upper,
                'bounds': value._bounds}

    def get_initial(self):
        initial = super().get_initial()
        return self.to_representation(initial)

if postgres_fields:
    class IntegerRangeField(RangeField):
        child = IntegerField()
        range_type = NumericRange

    class FloatRangeField(RangeField):
        child = FloatField()
        range_type = NumericRange

    class DecimalRangeField(RangeField):
        child = DecimalField(max_digits=None, decimal_places=None)
        range_type = NumericRange

    class DateTimeRangeField(RangeField):
        child = DateTimeField()
        range_type = DateTimeTZRange

    class DateRangeField(RangeField):
        child = DateField()
        range_type = DateRange

    # monkey patch modelserializer to map Native django Range fields to
    # drf_extra_fiels's Range fields.

    ModelSerializer.serializer_field_mapping[postgres_fields.DateTimeRangeField] = DateTimeRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.DateRangeField] = DateRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.IntegerRangeField] = IntegerRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.DecimalRangeField] = DecimalRangeField
    if compat.FloatRangeField:
        ModelSerializer.serializer_field_mapping[compat.FloatRangeField] = FloatRangeField

class LowercaseEmailField(EmailField):
    """
    An enhancement over django-rest-framework's EmailField to allow
    case-insensitive serialization and deserialization of e-mail addresses.
    """
    def to_internal_value(self, data):
        data = super(LowercaseEmailField, self).to_internal_value(data)
        return data.lower()

    def to_representation(self, value):
        value = super(LowercaseEmailField, self).to_representation(value)
        return value.lower()
<|endoftext|>


<commit_before>import imghdr
import io
import base64
import binascii
import uuid
import copy

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.utils import six
from django.utils.translation import ugettext_lazy as _

from rest_framework.fields import (
    DateField,
    DateTimeField,
    DictField,
    FileField,
    FloatField,
    ImageField,
    IntegerField,
)
from rest_framework.utils import html
from .compat import (
    DateRange,
    DateTimeTZRange,
    NumericRange,
    postgres_fields,
)

DEFAULT_CONTENT_TYPE = "application/octet-stream"

class Base64FieldMixin(object):
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError

    @property
    def INVALID_FILE_MESSAGE(self):
        raise NotImplementedError

    @property
    def INVALID_TYPE_MESSAGE(self):
        raise NotImplementedError

    EMPTY_VALUES = (None, '', [], (), {})

    def __init__(self, *args, **kwargs):
        self.represent_in_base64 = kwargs.pop('represent_in_base64', False)
        super(Base64FieldMixin, self).__init__(*args, **kwargs)

    def to_internal_value(self, base64_data):
        # Check if this is a base64 string
        if base64_data in self.EMPTY_VALUES:
            return None

        if isinstance(base64_data, six.string_types):
            # Strip base64 header.
            if ';base64,' in base64_data:
                header, base64_data = base64_data.split(';base64,')

            # Try to decode the file. Return validation error if it fails.
            try:
                decoded_file = base64.b64decode(base64_data)
            except (TypeError, binascii.Error, ValueError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)
            # Generate file name:
            file_name = self.get_file_name(decoded_file)
            # Get the file name extension:
            file_extension = self.get_file_extension(file_name, decoded_file)
            if file_extension not in self.ALLOWED_TYPES:
                raise ValidationError(self.INVALID_TYPE_MESSAGE)
            complete_file_name = file_name + "." + file_extension
            data = ContentFile(decoded_file, name=complete_file_name)
            return super(Base64FieldMixin, self).to_internal_value(data)
        raise ValidationError(_('Invalid type. This is not an base64 string: {}'.format(
            type(base64_data))))

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError

    def get_file_name(self, decoded_file):
        return str(uuid.uuid4())

    def to_representation(self, file):
        if self.represent_in_base64:
            # If the underlying ImageField is blank, a ValueError would be
            # raised on `open`. When representing as base64, simply return an
            # empty base64 str rather than let the exception propagate unhandled
            # up into serializers.
            if not file:
                return ''

            try:
                with open(file.path, 'rb') as f:
                    return base64.b64encode(f.read()).decode()
            except Exception:
                raise IOError("Error encoding file")
        else:
            return super(Base64FieldMixin, self).to_representation(file)

class Base64ImageField(Base64FieldMixin, ImageField):
    """
    A django-rest-framework field for handling image-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    ALLOWED_TYPES = (
        "jpeg",
        "jpg",
        "png",
        "gif"
    )
    INVALID_FILE_MESSAGE = _("Please upload a valid image.")
    INVALID_TYPE_MESSAGE = _("The type of the image couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        try:
            from PIL import Image
        except ImportError:
            raise ImportError("Pillow is not installed.")
        extension = imghdr.what(filename, decoded_file)

        # Try with PIL as fallback if format not detected due
        # to bug in imghdr https://bugs.python.org/issue16512
        if extension is None:
            image = Image.open(io.BytesIO(decoded_file))
            extension = image.format.lower()

        extension = "jpg" if extension == "jpeg" else extension
        return extension

class HybridImageField(Base64ImageField):
    """
    A django-rest-framework field for handling image-uploads through
    raw post data, with a fallback to multipart form data.
    """

    def to_internal_value(self, data):
        """
        Try Base64Field first, and then try the ImageField
        ``to_internal_value``, MRO doesn't work here because
        Base64FieldMixin throws before ImageField can run.
        """
        try:
            return Base64FieldMixin.to_internal_value(self, data)
        except ValidationError:
            return ImageField.to_internal_value(self, data)

class Base64FileField(Base64FieldMixin, FileField):
    """
    A django-rest-framework field for handling file-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError('List allowed file extensions')

    INVALID_FILE_MESSAGE = _("Please upload a valid file.")
    INVALID_TYPE_MESSAGE = _("The type of the file couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError('Implement file validation and return matching extension.')

class RangeField(DictField):

    range_type = None

    default_error_messages = {
        'not_a_dict': _('Expected a dictionary of items but got type "{input_type}".'),
        'too_much_content': _('Extra content not allowed "{extra}".'),
    }

    def to_internal_value(self, data):
        """
        Range instances <- Dicts of primitive datatypes.
        """
        initial_data = copy.deepcopy(self.parent.initial_data)
        if html.is_html_input(data):
            data = html.parse_html_dict(data)
        if not isinstance(data, dict):
            self.fail('not_a_dict', input_type=type(data).__name__)
        validated_dict = {}
        for key in ('lower', 'upper'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = self.child.run_validation(value)
        for key in ('bounds', 'empty'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = value
        if data:
            self.fail('too_much_content', extra=', '.join(map(str, data.keys())))

        self.parent.initial_data = initial_data
        return self.range_type(**validated_dict)

    def to_representation(self, value):
        """
        Range instances -> dicts of primitive datatypes.
        """
        if value.isempty:
            return {'empty': True}
        lower = self.child.to_representation(value.lower) if value.lower is not None else None
        upper = self.child.to_representation(value.upper) if value.upper is not None else None
        return {'lower': lower,
                'upper': upper,
                'bounds': value._bounds}

class IntegerRangeField(RangeField):
    child = IntegerField()
    range_type = NumericRange

class FloatRangeField(RangeField):
    child = FloatField()
    range_type = NumericRange

class DateTimeRangeField(RangeField):
    child = DateTimeField()
    range_type = DateTimeTZRange

class DateRangeField(RangeField):
    child = DateField()
    range_type = DateRange

if postgres_fields is not None:
    # monkey patch modelserializer to map Native django Range fields to
    # drf_extra_fiels's Range fields.
    from rest_framework.serializers import ModelSerializer
    ModelSerializer.serializer_field_mapping[postgres_fields.DateTimeRangeField] = DateTimeRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.DateRangeField] = DateRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.IntegerRangeField] = IntegerRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.FloatRangeField] = FloatRangeField
<commit_msg>Change preservering initial_data logic in RangeField<commit_after>import imghdr
import io
import base64
import binascii
import uuid
import copy

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.utils import six
from django.utils.translation import ugettext_lazy as _

from rest_framework.fields import (
    DateField,
    DateTimeField,
    DictField,
    FileField,
    FloatField,
    ImageField,
    IntegerField,
)
from rest_framework.utils import html
from .compat import (
    DateRange,
    DateTimeTZRange,
    NumericRange,
    postgres_fields,
)

DEFAULT_CONTENT_TYPE = "application/octet-stream"

class Base64FieldMixin(object):
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError

    @property
    def INVALID_FILE_MESSAGE(self):
        raise NotImplementedError

    @property
    def INVALID_TYPE_MESSAGE(self):
        raise NotImplementedError

    EMPTY_VALUES = (None, '', [], (), {})

    def __init__(self, *args, **kwargs):
        self.represent_in_base64 = kwargs.pop('represent_in_base64', False)
        super(Base64FieldMixin, self).__init__(*args, **kwargs)

    def to_internal_value(self, base64_data):
        # Check if this is a base64 string
        if base64_data in self.EMPTY_VALUES:
            return None

        if isinstance(base64_data, six.string_types):
            # Strip base64 header.
            if ';base64,' in base64_data:
                header, base64_data = base64_data.split(';base64,')

            # Try to decode the file. Return validation error if it fails.
            try:
                decoded_file = base64.b64decode(base64_data)
            except (TypeError, binascii.Error, ValueError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)
            # Generate file name:
            file_name = self.get_file_name(decoded_file)
            # Get the file name extension:
            file_extension = self.get_file_extension(file_name, decoded_file)
            if file_extension not in self.ALLOWED_TYPES:
                raise ValidationError(self.INVALID_TYPE_MESSAGE)
            complete_file_name = file_name + "." + file_extension
            data = ContentFile(decoded_file, name=complete_file_name)
            return super(Base64FieldMixin, self).to_internal_value(data)
        raise ValidationError(_('Invalid type. This is not an base64 string: {}'.format(
            type(base64_data))))

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError

    def get_file_name(self, decoded_file):
        return str(uuid.uuid4())

    def to_representation(self, file):
        if self.represent_in_base64:
            # If the underlying ImageField is blank, a ValueError would be
            # raised on `open`. When representing as base64, simply return an
            # empty base64 str rather than let the exception propagate unhandled
            # up into serializers.
            if not file:
                return ''

            try:
                with open(file.path, 'rb') as f:
                    return base64.b64encode(f.read()).decode()
            except Exception:
                raise IOError("Error encoding file")
        else:
            return super(Base64FieldMixin, self).to_representation(file)

class Base64ImageField(Base64FieldMixin, ImageField):
    """
    A django-rest-framework field for handling image-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    ALLOWED_TYPES = (
        "jpeg",
        "jpg",
        "png",
        "gif"
    )
    INVALID_FILE_MESSAGE = _("Please upload a valid image.")
    INVALID_TYPE_MESSAGE = _("The type of the image couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        try:
            from PIL import Image
        except ImportError:
            raise ImportError("Pillow is not installed.")
        extension = imghdr.what(filename, decoded_file)

        # Try with PIL as fallback if format not detected due
        # to bug in imghdr https://bugs.python.org/issue16512
        if extension is None:
            image = Image.open(io.BytesIO(decoded_file))
            extension = image.format.lower()

        extension = "jpg" if extension == "jpeg" else extension
        return extension

class HybridImageField(Base64ImageField):
    """
    A django-rest-framework field for handling image-uploads through
    raw post data, with a fallback to multipart form data.
    """

    def to_internal_value(self, data):
        """
        Try Base64Field first, and then try the ImageField
        ``to_internal_value``, MRO doesn't work here because
        Base64FieldMixin throws before ImageField can run.
        """
        try:
            return Base64FieldMixin.to_internal_value(self, data)
        except ValidationError:
            return ImageField.to_internal_value(self, data)

class Base64FileField(Base64FieldMixin, FileField):
    """
    A django-rest-framework field for handling file-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError('List allowed file extensions')

    INVALID_FILE_MESSAGE = _("Please upload a valid file.")
    INVALID_TYPE_MESSAGE = _("The type of the file couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError('Implement file validation and return matching extension.')

class RangeField(DictField):

    range_type = None

    default_error_messages = {
        'not_a_dict': _('Expected a dictionary of items but got type "{input_type}".'),
        'too_much_content': _('Extra content not allowed "{extra}".'),
    }

    def to_internal_value(self, data):
        """
        Range instances <- Dicts of primitive datatypes.
        """
        initial_data = copy.deepcopy(data)

        if html.is_html_input(data):
            data = html.parse_html_dict(data)
        if not isinstance(data, dict):
            self.fail('not_a_dict', input_type=type(data).__name__)
        validated_dict = {}
        for key in ('lower', 'upper'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = self.child.run_validation(value)
        for key in ('bounds', 'empty'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = value
        if data:
            self.fail('too_much_content', extra=', '.join(map(str, data.keys())))

        for key, value in initial_data.items():
            data[key] = value

        return self.range_type(**validated_dict)

    def to_representation(self, value):
        """
        Range instances -> dicts of primitive datatypes.
        """
        if value.isempty:
            return {'empty': True}
        lower = self.child.to_representation(value.lower) if value.lower is not None else None
        upper = self.child.to_representation(value.upper) if value.upper is not None else None
        return {'lower': lower,
                'upper': upper,
                'bounds': value._bounds}

class IntegerRangeField(RangeField):
    child = IntegerField()
    range_type = NumericRange

class FloatRangeField(RangeField):
    child = FloatField()
    range_type = NumericRange

class DateTimeRangeField(RangeField):
    child = DateTimeField()
    range_type = DateTimeTZRange

class DateRangeField(RangeField):
    child = DateField()
    range_type = DateRange

if postgres_fields is not None:
    # monkey patch modelserializer to map Native django Range fields to
    # drf_extra_fiels's Range fields.
    from rest_framework.serializers import ModelSerializer
    ModelSerializer.serializer_field_mapping[postgres_fields.DateTimeRangeField] = DateTimeRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.DateRangeField] = DateRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.IntegerRangeField] = IntegerRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.FloatRangeField] = FloatRangeField
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import sys
import socket
import platform
import datetime

IS_64BIT = sys.maxsize > 2 ** 32
try:
    from multiprocessing import cpu_count
    CPU_COUNT = cpu_count()
except:
    CPU_COUNT = None

HAVE_READLINE = True
try:
    import readline
except:
    HAVE_READLINE = False

try:
    import resource
except ImportError:
    resource = None

from application import Application, NullRoute, RESERVED_ARGS
from sinter import getargspec, inject
from render import render_json, AshesRenderFactory
from static import StaticApplication
from utils import bytes2human

from middleware.url import ScriptRootMiddleware
from middleware.context import SimpleContextProcessor

_CUR_PATH = os.path.dirname(os.path.abspath(__file__))
_ASSET_PATH = os.path.join(_CUR_PATH, '_clastic_assets')

# TODO: nominal SLA vs real/sampled SLA

DEFAULT_PAGE_TITLE = 'Clastic'

def create_app(page_title=DEFAULT_PAGE_TITLE):
    routes = [('/', get_all_meta_info, 'meta_base.html'),
              ('/clastic_assets/', StaticApplication(_ASSET_PATH)),
              ('/json/', get_all_meta_info, render_json)]
    resources = {'_meta_start_time': datetime.datetime.utcnow(),
                 'page_title': page_title}
    arf = AshesRenderFactory(_CUR_PATH, keep_whitespace=False)
    middlewares = [ScriptRootMiddleware(),
                   SimpleContextProcessor('script_root')]
    app = Application(routes, resources, middlewares, arf)
    return app

def get_all_meta_info(_application, _meta_start_time, page_title):
    app = _application
    ret = {}
    ret['routes'] = get_route_infos(app)
    ret['resources'] = get_resource_info(app)
    ret['env'] = get_env_info()

    mw_infos = []
    for mw in app.middlewares:
        mw_infos.append(get_mw_info(mw))
    ret['middlewares'] = mw_infos

    ret['abs_start_time'] = str(_meta_start_time)
    ret['rel_start_time'] = _rel_datetime(_meta_start_time)
    ret['page_title'] = page_title
    return ret

def get_route_infos(_application):
    app = _application
    ret = []
    for r in app.routes:
        if isinstance(r, NullRoute):
            continue
        r_info = {}
        r_info['url_pattern'] = r.pattern
        r_info['url_regex_pattern'] = r.regex.pattern
        r_info['endpoint'] = get_endpoint_info(r)
        r_info['render'] = get_render_info(r)
        r_info['args'] = get_route_arg_info(r)
        ret.append(r_info)
    return ret

def _trunc(str_val, length=70, trailer='...'):
    if len(str_val) > length:
        if trailer:
            str_val = str_val[:length - len(trailer)] + trailer
        else:
            str_val = str_val[:length]
    return str_val

def _rel_datetime(d, other=None):
    # TODO: add decimal rounding factor (default 0)
    if other is None:
        other = datetime.datetime.utcnow()
    diff = other - d
    s, days = diff.seconds, diff.days
    if days > 7 or days < 0:
        return d.strftime('%d %b %y')
    elif days == 1:
        return '1 day ago'
    elif days > 1:
        return '{0} days ago'.format(diff.days)
    elif s < 5:
        return 'just now'
    elif s < 60:
        return '{0} seconds ago'.format(s)
    elif s < 120:
        return '1 minute ago'
    elif s < 3600:
        return '{0} minutes ago'.format(s / 60)
    elif s < 7200:
        return '1 hour ago'
    else:
        return '{0} hours ago'.format(s / 3600)

def get_env_info():
    ret = {}
    ret['proc'] = get_proc_info()
    ret['host'] = get_host_info()
    ret['pyvm'] = get_pyvm_info()
    return ret

def get_proc_info():
    ret = {}
    ret['pid'] = os.getpid()
    _user_t, _sys_t = os.times()[:2]
    ret['cpu_times'] = {'user_time': _user_t, 'sys_time': _sys_t}
    ret['cwd'] = os.getcwdu()
    ret['umask'] = os.umask(os.umask(2))  # have to set to get
    ret['umask_str'] = '{0:03o}'.format(ret['umask'])
    try:
        import getpass
        ret['owner'] = getpass.getuser()
    except:
        try:
            ret['owner'] = os.getuid()
        except:
            pass
    try:
        # unix-only
        ret['ppid'] = os.getppid()
        ret['pgid'] = os.getpgrp()
        # add 0 to get current nice
        # also, seems to return process group's nice level
        ret['niceness'] = os.nice(0)
    except AttributeError:
        pass
    ret['rusage'] = get_rusage_dict()
    ret['rlimit'] = get_rlimit_dict()
    return ret

def get_rlimit_dict():
    ret = {}
    if not resource:
        return ret
    rname_val = [(rn[7:].lower(), val) for rn, val in resource.__dict__.items()
                 if rn.startswith('RLIMIT_')]
    for rn, val in rname_val:
        ret[rn] = resource.getrlimit(val)
    return ret

# TODO: byte order, path, prefix
def get_host_info():
    ret = {}
    ret['hostname'] = socket.gethostname()
    ret['hostfqdn'] = socket.getfqdn()
    ret['uname'] = platform.uname()
    ret['cpu_count'] = CPU_COUNT
    ret['platform'] = platform.platform()
    ret['platform_terse'] = platform.platform(terse=True)
    try:
        ret['load_avgs'] = os.getloadavg()
    except AttributeError:
        pass
    return ret

def get_rusage_dict(children=False):
    # TODO:
    # number of child processes?
    # page size
    # difference between a page out and a major fault?
    # NOTE: values commented with an * are untracked on Linux
    if not resource:
        return {}
    who = resource.RUSAGE_SELF
    if children:
        who = resource.RUSAGE_CHILDREN
    rr = resource.getrusage(who)
    max_rss_human = bytes2human(rr.ru_maxrss * 1024, ndigits=1)

    ret = {'cpu_times': {'user_time': rr.ru_utime,
                         'sys_time': rr.ru_stime},
           'memory': {'max_rss_human': max_rss_human,
                      'max_rss': rr.ru_maxrss,
                      'shared_rss': rr.ru_ixrss,    # *
                      'unshared_rss': rr.ru_idrss,  # *
                      'stack_rss': rr.ru_isrss},    # *
           'page_faults': {'minor_faults': rr.ru_minflt,
                           'major_faults': rr.ru_majflt,
                           'page_outs': rr.ru_nswap},  # *
           'blocking_io': {'input_ops': rr.ru_inblock,
                           'output_ops': rr.ru_oublock},
           'messages': {'sent': rr.ru_msgsnd,  # *
                        'received': rr.ru_msgrcv},  # *
           'signals': {'received': rr.ru_nsignals},  # *
           'ctx_switches': {'voluntary': rr.ru_nvcsw,
                            'involuntary': rr.ru_nivcsw}}
    return ret

def get_pyvm_info():
    ret = {}
    ret['executable'] = sys.executable
    ret['is_64bit'] = IS_64BIT
    ret['version'] = sys.version
    ret['version_info'] = list(sys.version_info)
    ret['have_ucs4'] = getattr(sys, 'maxunicode', 0) > 65536
    ret['have_readline'] = HAVE_READLINE
    try:
        ret['active_thread_count'] = len(sys._current_frames())
    except:
        ret['active_thread_count'] = None
    ret['recursion_limit'] = sys.getrecursionlimit()  # TODO: max_stack_depth?
    try:
        ret['gc'] = get_gc_info()
    except:
        pass
    ret['check_interval'] = sys.getcheckinterval()
    return ret

def get_gc_info():
    import gc
    ret = {}
    ret['is_enabled'] = gc.isenabled()
    ret['thresholds'] = gc.get_threshold()
    try:
        ret['counts'] = gc.get_count()
    except:
        pass
    try:
        ret['obj_count'] = len(gc.get_objects())
    except:
        pass
    return ret

def get_resource_info(_application):
    ret = []
    for key, val in _application.resources.items():
        trunc_val = _trunc(repr(val))
        ret.append({'key': key, 'value': trunc_val})
    return ret

def get_mw_info(mw):
    # TODO: what to do about render and endpoint provides?
    ret = {}
    ret['type_name'] = mw.__class__.__name__
    ret['provides'] = mw.provides
    ret['requires'] = mw.requires
    ret['repr'] = repr(mw)
    return ret

def get_endpoint_info(route):
    # TODO: callable object endpoints?
    ret = {}
    try:
        ret['name'] = route.endpoint.func_name
        ret['module_name'] = route.endpoint.__module__
    except:
        ret['name'] = repr(route.endpoint)
    return ret

def get_render_info(route):
    ret = {'type': None}
    render_arg = route.render_arg
    if route._render_factory and not callable(render_arg):
        ret['type'] = route._render_factory.__class__.__name__
        ret['arg'] = render_arg
    elif render_arg is None:
        ret['arg'] = None
    else:
        try:
            ret['arg'] = render_arg.func_name
        except AttributeError:
            ret['arg'] = render_arg.__class__.__name__
    return ret

def get_route_arg_info(route):
    r_args, _, _, r_defaults = getargspec(route.endpoint)
    r_defaults = dict(reversed(zip(reversed(r_args),
                                   reversed(r_defaults or []))))
    arg_srcs = []
    for arg in r_args:
        arg_src = {'name': arg}
        source = None
        if arg in RESERVED_ARGS:
            source = 'builtin'
        elif arg in route.path_args:
            source = 'url'
        elif arg in route._resources:
            source = 'resources'
        else:
            for mw in route._middlewares:
                if arg in mw.provides:
                    source = 'middleware'
                    break
        if source is None:
            if arg in r_defaults:
                source = 'default'
        arg_src['source'] = source
        arg_srcs.append(arg_src)
    # TODO: trace to application if middleware/resource
    return arg_srcs

MetaApplication = create_app()

class MetaPeripheral(object):
    title = 'Clastic MetaPeripheral'

    def get_general_items(self):
        "Returns list of 2-tuples to appear in the general section table"
        return []

    def get_all_items(self):
        return []

    def get_content(self):
        return None

    def get_help_text(self):
        return None

    def get_note_text(self):
        return None

    def get_extra_routes(self):
        return []

class BasicPeripheral(MetaPeripheral):
    title = 'Start Time'

    def get_general_items(self, _meta_application):
        start_time = _meta_application.resources['_meta_start_time']
        item = ('Start time', (_rel_datetime(start_time), start_time))
        return [item]

#class MetaSummarySection(object):
#    def __init__(self, title, subsections=None):
#        self.title = title
#        self.subsections = list(subsections or [])

class MetaApplication2(Application):
    def __init__(self, peripherals=None, page_title=DEFAULT_PAGE_TITLE):
        self.page_title = page_title
        self.peripherals = peripherals or []

        routes = [('/', self.get_main_page, 'meta2_base.html'),
                  ('/clastic_assets/', StaticApplication(_ASSET_PATH)),
                  ('/json/', self.get_main_page, render_json)]
        for peri in self.peripherals:
            routes.extend(peri.get_extra_routes())
        resources = {'_meta_start_time': datetime.datetime.utcnow(),
                     'page_title': page_title}
        arf = AshesRenderFactory(_CUR_PATH, keep_whitespace=False)
        mwares = [ScriptRootMiddleware(),
                  SimpleContextProcessor('script_root')]
        super(MetaApplication2, self).__init__(routes, resources, mwares, arf)

    def get_main_page(self, request, _application, _route):
        context = {'page_title': self.page_title,
                   'sections': [],
                   'general': []}
        kwargs = {'request': request,
                  '_route': _route,
                  '_application': _application,
                  '_meta_application': self}
        for peri in self.peripherals:
            cur = {'title': peri.title,
                   'note_text': inject(peri.get_note_text, kwargs),
                   'help_text': inject(peri.get_help_text, kwargs),
                   'content': inject(peri.get_content, kwargs)}
            context['sections'].append(cur)

            gen_items = inject(peri.get_general_items, kwargs) or []
            gen_items = _process_items(gen_items)
            context['general'].extend(gen_items)
        return context

def _process_items(all_items):
    ret = []
    for i, item in enumerate(all_items):
        cur = {}
        try:
            key, value = item
        except:
            try:
                key, value = item[0], item[1:]
            except:
                value = ''
                try:
                    key = repr(item)
                except:
                    key = 'unreprable object %s' % object.__repr__(key)
        if isinstance(key, basestring):
            cur['key'] = key
        else:
            try:
                cur['key'] = unicode(key[0])
                cur['key_detail'] = unicode(key[1])
            except:
                cur['key'] = unicode(key)
        if isinstance(value, basestring):
            cur['value'] = value
        else:
            try:
                cur['value'] = unicode(value[0])
                cur['value_detail'] = unicode(value[1])
            except:
                cur['value'] = str(value)
        ret.append(cur)
    return ret

if __name__ == '__main__':
    pass
<commit_msg>syncing MetaApplication, starting yet another new approach<commit_after># -*- coding: utf-8 -*-

import os
import sys
import socket
import platform
import datetime

IS_64BIT = sys.maxsize > 2 ** 32
try:
    from multiprocessing import cpu_count
    CPU_COUNT = cpu_count()
except:
    CPU_COUNT = None

HAVE_READLINE = True
try:
    import readline
except:
    HAVE_READLINE = False

try:
    import resource
except ImportError:
    resource = None

from application import Application, NullRoute, RESERVED_ARGS
from sinter import getargspec, inject
from render import render_json, AshesRenderFactory
from static import StaticApplication
from utils import bytes2human

from middleware.url import ScriptRootMiddleware
from middleware.context import SimpleContextProcessor

_CUR_PATH = os.path.dirname(os.path.abspath(__file__))
_ASSET_PATH = os.path.join(_CUR_PATH, '_clastic_assets')

# TODO: nominal SLA vs real/sampled SLA

DEFAULT_PAGE_TITLE = 'Clastic'

def create_app(page_title=DEFAULT_PAGE_TITLE):
    routes = [('/', get_all_meta_info, 'meta_base.html'),
              ('/clastic_assets/', StaticApplication(_ASSET_PATH)),
              ('/json/', get_all_meta_info, render_json)]
    resources = {'_meta_start_time': datetime.datetime.utcnow(),
                 'page_title': page_title}
    arf = AshesRenderFactory(_CUR_PATH, keep_whitespace=False)
    middlewares = [ScriptRootMiddleware(),
                   SimpleContextProcessor('script_root')]
    app = Application(routes, resources, middlewares, arf)
    return app

def get_all_meta_info(_application, _meta_start_time, page_title):
    app = _application
    ret = {}
    ret['routes'] = get_route_infos(app)
    ret['resources'] = get_resource_info(app)
    ret['env'] = get_env_info()

    mw_infos = []
    for mw in app.middlewares:
        mw_infos.append(get_mw_info(mw))
    ret['middlewares'] = mw_infos

    ret['abs_start_time'] = str(_meta_start_time)
    ret['rel_start_time'] = _rel_datetime(_meta_start_time)
    ret['page_title'] = page_title
    return ret

def get_route_infos(_application):
    app = _application
    ret = []
    for r in app.routes:
        if isinstance(r, NullRoute):
            continue
        r_info = {}
        r_info['url_pattern'] = r.pattern
        r_info['url_regex_pattern'] = r.regex.pattern
        r_info['endpoint'] = get_endpoint_info(r)
        r_info['render'] = get_render_info(r)
        r_info['args'] = get_route_arg_info(r)
        ret.append(r_info)
    return ret

def _trunc(str_val, length=70, trailer='...'):
    if len(str_val) > length:
        if trailer:
            str_val = str_val[:length - len(trailer)] + trailer
        else:
            str_val = str_val[:length]
    return str_val

def _rel_datetime(d, other=None):
    # TODO: add decimal rounding factor (default 0)
    if other is None:
        other = datetime.datetime.utcnow()
    diff = other - d
    s, days = diff.seconds, diff.days
    if days > 7 or days < 0:
        return d.strftime('%d %b %y')
    elif days == 1:
        return '1 day ago'
    elif days > 1:
        return '{0} days ago'.format(diff.days)
    elif s < 5:
        return 'just now'
    elif s < 60:
        return '{0} seconds ago'.format(s)
    elif s < 120:
        return '1 minute ago'
    elif s < 3600:
        return '{0} minutes ago'.format(s / 60)
    elif s < 7200:
        return '1 hour ago'
    else:
        return '{0} hours ago'.format(s / 3600)

def get_env_info():
    ret = {}
    ret['proc'] = get_proc_info()
    ret['host'] = get_host_info()
    ret['pyvm'] = get_pyvm_info()
    return ret

def get_proc_info():
    ret = {}
    ret['pid'] = os.getpid()
    _user_t, _sys_t = os.times()[:2]
    ret['cpu_times'] = {'user_time': _user_t, 'sys_time': _sys_t}
    ret['cwd'] = os.getcwdu()
    ret['umask'] = os.umask(os.umask(2))  # have to set to get
    ret['umask_str'] = '{0:03o}'.format(ret['umask'])
    try:
        import getpass
        ret['owner'] = getpass.getuser()
    except:
        try:
            ret['owner'] = os.getuid()
        except:
            pass
    try:
        # unix-only
        ret['ppid'] = os.getppid()
        ret['pgid'] = os.getpgrp()
        # add 0 to get current nice
        # also, seems to return process group's nice level
        ret['niceness'] = os.nice(0)
    except AttributeError:
        pass
    ret['rusage'] = get_rusage_dict()
    ret['rlimit'] = get_rlimit_dict()
    return ret

def get_rlimit_dict():
    ret = {}
    if not resource:
        return ret
    rname_val = [(rn[7:].lower(), val) for rn, val in resource.__dict__.items()
                 if rn.startswith('RLIMIT_')]
    for rn, val in rname_val:
        ret[rn] = resource.getrlimit(val)
    return ret

# TODO: byte order, path, prefix
def get_host_info():
    ret = {}
    ret['hostname'] = socket.gethostname()
    ret['hostfqdn'] = socket.getfqdn()
    ret['uname'] = platform.uname()
    ret['cpu_count'] = CPU_COUNT
    ret['platform'] = platform.platform()
    ret['platform_terse'] = platform.platform(terse=True)
    try:
        ret['load_avgs'] = os.getloadavg()
    except AttributeError:
        pass
    return ret

def get_rusage_dict(children=False):
    # TODO:
    # number of child processes?
    # page size
    # difference between a page out and a major fault?
    # NOTE: values commented with an * are untracked on Linux
    if not resource:
        return {}
    who = resource.RUSAGE_SELF
    if children:
        who = resource.RUSAGE_CHILDREN
    rr = resource.getrusage(who)
    max_rss_human = bytes2human(rr.ru_maxrss * 1024, ndigits=1)

    ret = {'cpu_times': {'user_time': rr.ru_utime,
                         'sys_time': rr.ru_stime},
           'memory': {'max_rss_human': max_rss_human,
                      'max_rss': rr.ru_maxrss,
                      'shared_rss': rr.ru_ixrss,    # *
                      'unshared_rss': rr.ru_idrss,  # *
                      'stack_rss': rr.ru_isrss},    # *
           'page_faults': {'minor_faults': rr.ru_minflt,
                           'major_faults': rr.ru_majflt,
                           'page_outs': rr.ru_nswap},  # *
           'blocking_io': {'input_ops': rr.ru_inblock,
                           'output_ops': rr.ru_oublock},
           'messages': {'sent': rr.ru_msgsnd,  # *
                        'received': rr.ru_msgrcv},  # *
           'signals': {'received': rr.ru_nsignals},  # *
           'ctx_switches': {'voluntary': rr.ru_nvcsw,
                            'involuntary': rr.ru_nivcsw}}
    return ret

def get_pyvm_info():
    ret = {}
    ret['executable'] = sys.executable
    ret['is_64bit'] = IS_64BIT
    ret['version'] = sys.version
    ret['version_info'] = list(sys.version_info)
    ret['have_ucs4'] = getattr(sys, 'maxunicode', 0) > 65536
    ret['have_readline'] = HAVE_READLINE
    try:
        ret['active_thread_count'] = len(sys._current_frames())
    except:
        ret['active_thread_count'] = None
    ret['recursion_limit'] = sys.getrecursionlimit()  # TODO: max_stack_depth?
    try:
        ret['gc'] = get_gc_info()
    except:
        pass
    ret['check_interval'] = sys.getcheckinterval()
    return ret

def get_gc_info():
    import gc
    ret = {}
    ret['is_enabled'] = gc.isenabled()
    ret['thresholds'] = gc.get_threshold()
    try:
        ret['counts'] = gc.get_count()
    except:
        pass
    try:
        ret['obj_count'] = len(gc.get_objects())
    except:
        pass
    return ret

def get_resource_info(_application):
    ret = []
    for key, val in _application.resources.items():
        trunc_val = _trunc(repr(val))
        ret.append({'key': key, 'value': trunc_val})
    return ret

def get_mw_info(mw):
    # TODO: what to do about render and endpoint provides?
    ret = {}
    ret['type_name'] = mw.__class__.__name__
    ret['provides'] = mw.provides
    ret['requires'] = mw.requires
    ret['repr'] = repr(mw)
    return ret

def get_endpoint_info(route):
    # TODO: callable object endpoints?
    ret = {}
    try:
        ret['name'] = route.endpoint.func_name
        ret['module_name'] = route.endpoint.__module__
    except:
        ret['name'] = repr(route.endpoint)
    return ret

def get_render_info(route):
    ret = {'type': None}
    render_arg = route.render_arg
    if route._render_factory and not callable(render_arg):
        ret['type'] = route._render_factory.__class__.__name__
        ret['arg'] = render_arg
    elif render_arg is None:
        ret['arg'] = None
    else:
        try:
            ret['arg'] = render_arg.func_name
        except AttributeError:
            ret['arg'] = render_arg.__class__.__name__
    return ret

def get_route_arg_info(route):
    r_args, _, _, r_defaults = getargspec(route.endpoint)
    r_defaults = dict(reversed(zip(reversed(r_args),
                                   reversed(r_defaults or []))))
    arg_srcs = []
    for arg in r_args:
        arg_src = {'name': arg}
        source = None
        if arg in RESERVED_ARGS:
            source = 'builtin'
        elif arg in route.path_args:
            source = 'url'
        elif arg in route._resources:
            source = 'resources'
        else:
            for mw in route._middlewares:
                if arg in mw.provides:
                    source = 'middleware'
                    break
        if source is None:
            if arg in r_defaults:
                source = 'default'
        arg_src['source'] = source
        arg_srcs.append(arg_src)
    # TODO: trace to application if middleware/resource
    return arg_srcs

MetaApplication = create_app()

class MetaPeripheral(object):
    title = 'Clastic MetaPeripheral'
    group_key = 'mp'

    def get_general_items(self):
        "Returns list of 2-tuples to appear in the general section table"
        return []

    def get_all_items(self):
        return self.get_general_items()

    def get_content(self):
        return None

    def get_help_text(self):
        return None

    def get_note_text(self):
        return None

    def get_extra_routes(self):
        return []

class BasicPeripheral(MetaPeripheral):
    title = 'Start Time'
    group_key = 'basic'

    def get_general_items(self, _meta_application):
        start_time = _meta_application.resources['_meta_start_time']
        item = ('Start time', (_rel_datetime(start_time), start_time))
        return [item]

#class MetaSummarySection(object):
#    def __init__(self, title, subsections=None):
#        self.title = title
#        self.subsections = list(subsections or [])

def _autohuman(obj):
    try:
        text = unicode(obj).replace('_', ' ').strip().capitalize()
    except:
        try:
            text = repr(obj)
        except:
            text = 'unreprable object %s' % object.__repr__(obj)
    return text

class Item(object):
    def __init__(self, key, value,
                 key_detail=None, value_detail=None,
                 key_human=None, value_human=None):
        self.key = key
        self.value = value
        self.key_detail = key_detail or ''
        self.value_detail = value_detail or ''
        self.key_human = key_human or _autohuman(key)
        self.value_human = value_human or _autohuman(value)

class MetaApplication2(Application):
    def __init__(self, peripherals=None, page_title=DEFAULT_PAGE_TITLE):
        self.page_title = page_title
        self.peripherals = peripherals or []

        routes = [('/', self.get_main_page, 'meta2_base.html'),
                  ('/clastic_assets/', StaticApplication(_ASSET_PATH)),
                  ('/json/', self.get_main_page, render_json)]
        for peri in self.peripherals:
            routes.extend(peri.get_extra_routes())
        resources = {'_meta_start_time': datetime.datetime.utcnow(),
                     'page_title': page_title}
        arf = AshesRenderFactory(_CUR_PATH, keep_whitespace=False)
        mwares = [ScriptRootMiddleware(),
                  SimpleContextProcessor('script_root')]
        super(MetaApplication2, self).__init__(routes, resources, mwares, arf)

    def get_main(self):  # let's try this again
        pass

    def get_main_page(self, request, _application, _route):
        context = {'page_title': self.page_title,
                   'sections': [],
                   'general': []}
        kwargs = {'request': request,
                  '_route': _route,
                  '_application': _application,
                  '_meta_application': self}
        for peri in self.peripherals:
            cur = {'title': peri.title,
                   'note_text': inject(peri.get_note_text, kwargs),
                   'help_text': inject(peri.get_help_text, kwargs),
                   'content': inject(peri.get_content, kwargs)}
            context[peri.group_key] = cur
            #context['sections'].append(cur)

            gen_items = inject(peri.get_general_items, kwargs) or []
            gen_items = _process_items(gen_items)
            context['general'].extend(gen_items)
        return context

class EnvironmentPeripheral(MetaPeripheral):
    title = 'Environment'
    group_key = 'env'

    # general items: load averages

    def get_context(self):
        return get_env_info()

def _process_items(all_items):
    """ Really, each key/value/key detail/value detail should have a
    human readable form and a machine readable form. That's a lot of
    keys, should probably do that later.
    """
    ret = []
    for i, item in enumerate(all_items):
        cur = {}
        try:
            key, value = item
        except:
            try:
                key, value = item[0], item[1:]
            except:
                value = ''
                try:
                    key = repr(item)
                except:
                    key = 'unreprable object %s' % object.__repr__(key)
        if isinstance(key, basestring):
            cur['key'] = key
        else:
            try:
                cur['key'] = unicode(key[0])
                cur['key_detail'] = unicode(key[1])
            except:
                cur['key'] = unicode(key)
        if isinstance(value, basestring):
            cur['value'] = value
        else:
            try:
                cur['value'] = unicode(value[0])
                cur['value_detail'] = unicode(value[1])
            except:
                cur['value'] = str(value)
        ret.append(cur)
    return ret

"""
* register template, get keys
"""

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>import base64
import binascii
import imghdr
import uuid

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.utils import six
from django.utils.translation import ugettext_lazy as _

from rest_framework.fields import (
    DateField,
    DateTimeField,
    DictField,
    FileField,
    FloatField,
    ImageField,
    IntegerField,
)
from rest_framework.utils import html
from .compat import (
    DateRange,
    DateTimeTZRange,
    NumericRange,
    postgres_fields,
)

DEFAULT_CONTENT_TYPE = "application/octet-stream"

class Base64FieldMixin(object):
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError

    @property
    def INVALID_FILE_MESSAGE(self):
        raise NotImplementedError

    @property
    def INVALID_TYPE_MESSAGE(self):
        raise NotImplementedError

    EMPTY_VALUES = (None, '', [], (), {})

    def __init__(self, *args, **kwargs):
        self.represent_in_base64 = kwargs.pop('represent_in_base64', False)
        super(Base64FieldMixin, self).__init__(*args, **kwargs)

    def to_internal_value(self, base64_data):
        # Check if this is a base64 string
        if base64_data in self.EMPTY_VALUES:
            return None

        if isinstance(base64_data, six.string_types):
            # Strip base64 header.
            if ';base64,' in base64_data:
                header, base64_data = base64_data.split(';base64,')

            # Try to decode the file. Return validation error if it fails.
            try:
                decoded_file = base64.b64decode(base64_data)
            except (TypeError, binascii.Error, ValueError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)
            # Generate file name:
            file_name = str(uuid.uuid4())[:12]  # 12 characters are more than enough.
            # Get the file name extension:
            file_extension = self.get_file_extension(file_name, decoded_file)
            if file_extension not in self.ALLOWED_TYPES:
                raise ValidationError(self.INVALID_TYPE_MESSAGE)
            complete_file_name = file_name + "." + file_extension
            data = ContentFile(decoded_file, name=complete_file_name)
            return super(Base64FieldMixin, self).to_internal_value(data)
        raise ValidationError(_('This is not an base64 string'))

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError

    def to_representation(self, file):
        if self.represent_in_base64:
            try:
                with open(file.path, 'rb') as f:
                    return base64.b64encode(f.read()).decode()
            except Exception:
                raise IOError("Error encoding file")
        else:
            return super(Base64FieldMixin, self).to_representation(file)

class Base64ImageField(Base64FieldMixin, ImageField):
    """
    A django-rest-framework field for handling image-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    ALLOWED_TYPES = (
        "jpeg",
        "jpg",
        "png",
        "gif"
    )
    INVALID_FILE_MESSAGE = _("Please upload a valid image.")
    INVALID_TYPE_MESSAGE = _("The type of the image couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        extension = imghdr.what(filename, decoded_file)
        extension = "jpg" if extension == "jpeg" else extension
        return extension

class HybridImageField(Base64ImageField):
    """
    A django-rest-framework field for handling image-uploads through
    raw post data, with a fallback to multipart form data.
    """

    def to_internal_value(self, data):
        """
        Try Base64Field first, and then try the ImageField
        ``to_internal_value``, MRO doesn't work here because
        Base64FieldMixin throws before ImageField can run.
        """
        try:
            return Base64FieldMixin.to_internal_value(self, data)
        except ValidationError:
            return ImageField.to_internal_value(self, data)

class Base64FileField(Base64FieldMixin, FileField):
    """
    A django-rest-framework field for handling file-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError('List allowed file extensions')

    INVALID_FILE_MESSAGE = _("Please upload a valid file.")
    INVALID_TYPE_MESSAGE = _("The type of the file couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError('Implement file validation and return matching extension.')

class RangeField(DictField):

    range_type = None

    default_error_messages = {
        'not_a_dict': _('Expected a dictionary of items but got type "{input_type}".'),
        'too_much_content': _('Extra content not allowed "{extra}".'),
    }

    def to_internal_value(self, data):
        """
        Range instances <- Dicts of primitive datatypes.
        """
        if html.is_html_input(data):
            data = html.parse_html_dict(data)
        if not isinstance(data, dict):
            self.fail('not_a_dict', input_type=type(data).__name__)
        validated_dict = {}
        for key in ('lower', 'upper'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = self.child.run_validation(value)
        for key in ('bounds', 'empty'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = value
        if data:
            self.fail('too_much_content', extra=', '.join(map(str, data.keys())))
        return self.range_type(**validated_dict)

    def to_representation(self, value):
        """
        Range instances -> dicts of primitive datatypes.
        """
        if value.isempty:
            return {'empty': True}
        lower = self.child.to_representation(value.lower) if value.lower is not None else None
        upper = self.child.to_representation(value.upper) if value.upper is not None else None
        return {'lower': lower,
                'upper': upper,
                'bounds': value._bounds}

class IntegerRangeField(RangeField):
    child = IntegerField()
    range_type = NumericRange

class FloatRangeField(RangeField):
    child = FloatField()
    range_type = NumericRange

class DateTimeRangeField(RangeField):
    child = DateTimeField()
    range_type = DateTimeTZRange

class DateRangeField(RangeField):
    child = DateField()
    range_type = DateRange

if postgres_fields is not None:
    # monkey patch modelserializer to map Native django Range fields to
    # drf_extra_fiels's Range fields.
    from rest_framework.serializers import ModelSerializer
    ModelSerializer.serializer_field_mapping[postgres_fields.DateTimeRangeField] = DateTimeRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.DateRangeField] = DateRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.IntegerRangeField] = IntegerRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.FloatRangeField] = FloatRangeField
<commit_msg>Replaced buggy imghdr with PIL<commit_after>import io
import base64
import binascii
import uuid

from PIL import Image

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.utils import six
from django.utils.translation import ugettext_lazy as _

from rest_framework.fields import (
    DateField,
    DateTimeField,
    DictField,
    FileField,
    FloatField,
    ImageField,
    IntegerField,
)
from rest_framework.utils import html
from .compat import (
    DateRange,
    DateTimeTZRange,
    NumericRange,
    postgres_fields,
)

DEFAULT_CONTENT_TYPE = "application/octet-stream"

class Base64FieldMixin(object):
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError

    @property
    def INVALID_FILE_MESSAGE(self):
        raise NotImplementedError

    @property
    def INVALID_TYPE_MESSAGE(self):
        raise NotImplementedError

    EMPTY_VALUES = (None, '', [], (), {})

    def __init__(self, *args, **kwargs):
        self.represent_in_base64 = kwargs.pop('represent_in_base64', False)
        super(Base64FieldMixin, self).__init__(*args, **kwargs)

    def to_internal_value(self, base64_data):
        # Check if this is a base64 string
        if base64_data in self.EMPTY_VALUES:
            return None

        if isinstance(base64_data, six.string_types):
            # Strip base64 header.
            if ';base64,' in base64_data:
                header, base64_data = base64_data.split(';base64,')

            # Try to decode the file. Return validation error if it fails.
            try:
                decoded_file = base64.b64decode(base64_data)
            except (TypeError, binascii.Error, ValueError):
                raise ValidationError(self.INVALID_FILE_MESSAGE)
            # Generate file name:
            file_name = str(uuid.uuid4())[:12]  # 12 characters are more than enough.
            # Get the file name extension:
            file_extension = self.get_file_extension(file_name, decoded_file)
            if file_extension not in self.ALLOWED_TYPES:
                raise ValidationError(self.INVALID_TYPE_MESSAGE)
            complete_file_name = file_name + "." + file_extension
            data = ContentFile(decoded_file, name=complete_file_name)
            return super(Base64FieldMixin, self).to_internal_value(data)
        raise ValidationError(_('This is not an base64 string'))

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError

    def to_representation(self, file):
        if self.represent_in_base64:
            try:
                with open(file.path, 'rb') as f:
                    return base64.b64encode(f.read()).decode()
            except Exception:
                raise IOError("Error encoding file")
        else:
            return super(Base64FieldMixin, self).to_representation(file)

class Base64ImageField(Base64FieldMixin, ImageField):
    """
    A django-rest-framework field for handling image-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    ALLOWED_TYPES = (
        "jpeg",
        "jpg",
        "png",
        "gif"
    )
    INVALID_FILE_MESSAGE = _("Please upload a valid image.")
    INVALID_TYPE_MESSAGE = _("The type of the image couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        image = Image.open(io.BytesIO(decoded_file))
        extension = image.format.lower()
        extension = "jpg" if extension == "jpeg" else extension
        return extension

class HybridImageField(Base64ImageField):
    """
    A django-rest-framework field for handling image-uploads through
    raw post data, with a fallback to multipart form data.
    """

    def to_internal_value(self, data):
        """
        Try Base64Field first, and then try the ImageField
        ``to_internal_value``, MRO doesn't work here because
        Base64FieldMixin throws before ImageField can run.
        """
        try:
            return Base64FieldMixin.to_internal_value(self, data)
        except ValidationError:
            return ImageField.to_internal_value(self, data)

class Base64FileField(Base64FieldMixin, FileField):
    """
    A django-rest-framework field for handling file-uploads through raw post data.
    It uses base64 for en-/decoding the contents of the file.
    """
    @property
    def ALLOWED_TYPES(self):
        raise NotImplementedError('List allowed file extensions')

    INVALID_FILE_MESSAGE = _("Please upload a valid file.")
    INVALID_TYPE_MESSAGE = _("The type of the file couldn't be determined.")

    def get_file_extension(self, filename, decoded_file):
        raise NotImplementedError('Implement file validation and return matching extension.')

class RangeField(DictField):

    range_type = None

    default_error_messages = {
        'not_a_dict': _('Expected a dictionary of items but got type "{input_type}".'),
        'too_much_content': _('Extra content not allowed "{extra}".'),
    }

    def to_internal_value(self, data):
        """
        Range instances <- Dicts of primitive datatypes.
        """
        if html.is_html_input(data):
            data = html.parse_html_dict(data)
        if not isinstance(data, dict):
            self.fail('not_a_dict', input_type=type(data).__name__)
        validated_dict = {}
        for key in ('lower', 'upper'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = self.child.run_validation(value)
        for key in ('bounds', 'empty'):
            try:
                value = data.pop(key)
            except KeyError:
                continue
            validated_dict[six.text_type(key)] = value
        if data:
            self.fail('too_much_content', extra=', '.join(map(str, data.keys())))
        return self.range_type(**validated_dict)

    def to_representation(self, value):
        """
        Range instances -> dicts of primitive datatypes.
        """
        if value.isempty:
            return {'empty': True}
        lower = self.child.to_representation(value.lower) if value.lower is not None else None
        upper = self.child.to_representation(value.upper) if value.upper is not None else None
        return {'lower': lower,
                'upper': upper,
                'bounds': value._bounds}

class IntegerRangeField(RangeField):
    child = IntegerField()
    range_type = NumericRange

class FloatRangeField(RangeField):
    child = FloatField()
    range_type = NumericRange

class DateTimeRangeField(RangeField):
    child = DateTimeField()
    range_type = DateTimeTZRange

class DateRangeField(RangeField):
    child = DateField()
    range_type = DateRange

if postgres_fields is not None:
    # monkey patch modelserializer to map Native django Range fields to
    # drf_extra_fiels's Range fields.
    from rest_framework.serializers import ModelSerializer
    ModelSerializer.serializer_field_mapping[postgres_fields.DateTimeRangeField] = DateTimeRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.DateRangeField] = DateRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.IntegerRangeField] = IntegerRangeField
    ModelSerializer.serializer_field_mapping[postgres_fields.FloatRangeField] = FloatRangeField
<|endoftext|>


<commit_before># coding=utf-8
from datetime import datetime
from os.path import basename, join
import tempfile

from flask import (
    Flask, request, has_request_context, render_template, make_response
)
from jinja2 import ChoiceLoader, FileSystemLoader, PackageLoader
from markupsafe import Markup
from moar import FileStorage, Thumbnailer

from .jinja_includewith import IncludeWith
from .markdown_ext import MarkdownExtension
from .tglobals import active, ToC

APP_NAME = 'clay'

TEMPLATE_GLOBALS = {
    'CLAY_URL': 'http://lucuma.github.com/Clay',
    'active': active,
    'now': datetime.utcnow(),
    'dir': dir,
    'enumerate': enumerate,
    'map': map,
    'zip': zip,
}

class WSGIApplication(Flask):

    def __init__(self, source_dir, build_dir, thumbs_url):
        super(WSGIApplication, self).__init__(
            APP_NAME, template_folder=source_dir, static_folder=None)
        self.jinja_loader = get_jinja_loader(source_dir)
        self.jinja_options = get_jinja_options()

        tempdir = tempfile.gettempdir()
        self.tempdir = tempdir
        storage = FileStorage(tempdir, base_url=thumbs_url)
        self.thumbnailer = Thumbnailer(source_dir, storage=storage)
        TEMPLATE_GLOBALS['thumbnail'] = self.thumbnailer

        toc = ToC(source_dir)

        def render_toc(*args, **kwargs):
            html = toc(*args, **kwargs)
            return Markup(html)

        TEMPLATE_GLOBALS['toc'] = render_toc

        self.build_dir = build_dir
        self.context_processor(lambda: TEMPLATE_GLOBALS)
        self.debug = True

    def get_test_client(self, host, port):
        self.testing = True
        self.config['SERVER_NAME'] = '%s:%s' % (host, port)
        return self.test_client()

    def render_template(self, path, context, host, port):
        if has_request_context():
            context.update(request.values.to_dict())
            return render_template(path, **context)

        self.thumbnailer.echo = not path.startswith('_index')
        self.thumbnailer.thumbs_storage.base_url = '/'
        self.thumbnailer.thumbs_storage.out_path = self.build_dir
        with self.test_request_context(
                '/' + path, method='GET',
                base_url='http://%s:%s' % (host, port)):
            return render_template(path, **context)

    def get_thumb_fullpath(self, thumbpath):
        return join(self.thumbnailer.thumbs_storage.out_path, thumbpath)

    def response(self, content, status=200, mimetype='text/plain'):
        resp = make_response(content, status)
        resp.mimetype = mimetype
        return resp

def get_jinja_loader(source_dir):
    return ChoiceLoader([
        FileSystemLoader(source_dir),
        PackageLoader('clay', basename(source_dir)),
    ])

def get_jinja_options():
    return {
        'autoescape': True,
        'extensions': [MarkdownExtension, 'jinja2.ext.with_', IncludeWith],
    }
<commit_msg>Fix ImportError<commit_after># coding=utf-8
from datetime import datetime
from os.path import basename, join
import tempfile

from flask import (
    Flask, request, has_request_context, render_template, make_response
)
from jinja2 import ChoiceLoader, FileSystemLoader, PackageLoader
from markupsafe import Markup
from moar import Thumbnailer, Storage as FileStorage

from .jinja_includewith import IncludeWith
from .markdown_ext import MarkdownExtension
from .tglobals import active, ToC

APP_NAME = 'clay'

TEMPLATE_GLOBALS = {
    'CLAY_URL': 'http://lucuma.github.com/Clay',
    'active': active,
    'now': datetime.utcnow(),
    'dir': dir,
    'enumerate': enumerate,
    'map': map,
    'zip': zip,
}

class WSGIApplication(Flask):

    def __init__(self, source_dir, build_dir, thumbs_url):
        super(WSGIApplication, self).__init__(
            APP_NAME, template_folder=source_dir, static_folder=None)
        self.jinja_loader = get_jinja_loader(source_dir)
        self.jinja_options = get_jinja_options()

        tempdir = tempfile.gettempdir()
        self.tempdir = tempdir
        storage = FileStorage(tempdir, base_url=thumbs_url)
        self.thumbnailer = Thumbnailer(source_dir, storage=storage)
        TEMPLATE_GLOBALS['thumbnail'] = self.thumbnailer

        toc = ToC(source_dir)

        def render_toc(*args, **kwargs):
            html = toc(*args, **kwargs)
            return Markup(html)

        TEMPLATE_GLOBALS['toc'] = render_toc

        self.build_dir = build_dir
        self.context_processor(lambda: TEMPLATE_GLOBALS)
        self.debug = True

    def get_test_client(self, host, port):
        self.testing = True
        self.config['SERVER_NAME'] = '%s:%s' % (host, port)
        return self.test_client()

    def render_template(self, path, context, host, port):
        if has_request_context():
            context.update(request.values.to_dict())
            return render_template(path, **context)

        self.thumbnailer.echo = not path.startswith('_index')
        self.thumbnailer.thumbs_storage.base_url = '/'
        self.thumbnailer.thumbs_storage.out_path = self.build_dir
        with self.test_request_context(
                '/' + path, method='GET',
                base_url='http://%s:%s' % (host, port)):
            return render_template(path, **context)

    def get_thumb_fullpath(self, thumbpath):
        return join(self.thumbnailer.thumbs_storage.out_path, thumbpath)

    def response(self, content, status=200, mimetype='text/plain'):
        resp = make_response(content, status)
        resp.mimetype = mimetype
        return resp

def get_jinja_loader(source_dir):
    return ChoiceLoader([
        FileSystemLoader(source_dir),
        PackageLoader('clay', basename(source_dir)),
    ])

def get_jinja_options():
    return {
        'autoescape': True,
        'extensions': [MarkdownExtension, 'jinja2.ext.with_', IncludeWith],
    }
<|endoftext|>


<commit_before>__VERSION__ = '1.2.7'
<commit_msg>Bump version 1.2.8<commit_after>__VERSION__ = '1.2.8'
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
cli.autocert: provides entry point main().
'''

import os
import imp
import sys
import logging
import requests

from subprocess import check_output
from argparse import ArgumentParser, RawDescriptionHelpFormatter

try:
    from ruamel import yaml
    from urlpath import URL
    from attrdict import AttrDict
    from packaging.version import parse as version_parse
except ImportError as ie:
    print(ie)
    print('perhaps you need to install cli/requirements.txt via pip3')

from cli.utils.version import version as cli_version
from cli.output import output

VERSIONS = [
    'cli',
    'api'
]

class VersionCheckFailedError(Exception):
    def __init__(self, version, required):
        msg = 'auto-cert/api {version} is not at least {required}'.format(**locals())
        super(VersionCheckFailedError, self).__init__(msg)

def api_version(ns):
    response = requests.get(ns.api_url / 'version')
    version = 'unknown'
    if response.status_code == 200:
        version = response.json()['version']
    return version

def version_check(ns):
    if ns.version not in VERSIONS:
        version = api_version(ns)
        version = version.split('-')[0]
        if  version_parse(version) >= version_parse(cli_version()):
            logging.debug('version_check: PASSED')
            return
        raise VersionCheckFailedError(version, cli_version())
    if ns.version == VERSIONS[0]:
        output({'cli-version': cli_version()})
    elif ns.version == VERSIONS[1]:
        output({'api-version': api_version(ns)})
    sys.exit(0)

def add_subparsers(parser):
    '''
    add all files that end with _parser.py in the cli/ directory
    call 'add_parser', passing subparsers to each found module
    '''
    subparsers = parser.add_subparsers(
        dest='command',
        title='commands',
        description='choose a command to run')
    commands_path = os.path.dirname(__file__)
    for filename in [f for f in os.listdir(commands_path) if f.endswith('_command.py')]:
        command = imp.load_source(filename.split('.')[0], commands_path+'/'+filename)
        command.add_parser(subparsers)
    subparsers.required = True
    return subparsers

def main():
    parser = ArgumentParser(
        add_help=False)
    parser.add_argument(
        '--debug',
        action='store_true',
        help='turn on debug mode')
    parser.add_argument(
        '--version',
        choices = VERSIONS,
        const=VERSIONS[0],
        nargs='?',
        help='default=%(const)s show the version')
    parser.add_argument(
        '--verbose',
        action='store_true',
        help='more verbose output')
    parser.add_argument(
        '--api-url',
        metavar='URL',
        type=URL,
        default=r'http://0.0.0.0',
        help='default=%(default)s set the api url to use')

    ns, rem = parser.parse_known_args()
    if not any([h in rem for h in ('-h', '--help')]):
        version_check(ns)

    config = {}

    parser = ArgumentParser(
        parents=[parser],
        description=__doc__,
        formatter_class=RawDescriptionHelpFormatter,
        argument_default=config)

    add_subparsers(parser)

    ns = parser.parse_args()
    if ns.debug:
        print('ns =', ns)
    ns.func(ns)
<commit_msg>added semicolons after defaults in help messages<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
cli.autocert: provides entry point main().
'''

import os
import imp
import sys
import logging
import requests

from subprocess import check_output
from argparse import ArgumentParser, RawDescriptionHelpFormatter

try:
    from ruamel import yaml
    from urlpath import URL
    from attrdict import AttrDict
    from packaging.version import parse as version_parse
except ImportError as ie:
    print(ie)
    print('perhaps you need to install cli/requirements.txt via pip3')

from cli.utils.version import version as cli_version
from cli.output import output

VERSIONS = [
    'cli',
    'api'
]

class VersionCheckFailedError(Exception):
    def __init__(self, version, required):
        msg = 'auto-cert/api {version} is not at least {required}'.format(**locals())
        super(VersionCheckFailedError, self).__init__(msg)

def api_version(ns):
    response = requests.get(ns.api_url / 'version')
    version = 'unknown'
    if response.status_code == 200:
        version = response.json()['version']
    return version

def version_check(ns):
    if ns.version not in VERSIONS:
        version = api_version(ns)
        version = version.split('-')[0]
        if  version_parse(version) >= version_parse(cli_version()):
            logging.debug('version_check: PASSED')
            return
        raise VersionCheckFailedError(version, cli_version())
    if ns.version == VERSIONS[0]:
        output({'cli-version': cli_version()})
    elif ns.version == VERSIONS[1]:
        output({'api-version': api_version(ns)})
    sys.exit(0)

def add_subparsers(parser):
    '''
    add all files that end with _parser.py in the cli/ directory
    call 'add_parser', passing subparsers to each found module
    '''
    subparsers = parser.add_subparsers(
        dest='command',
        title='commands',
        description='choose a command to run')
    commands_path = os.path.dirname(__file__)
    for filename in [f for f in os.listdir(commands_path) if f.endswith('_command.py')]:
        command = imp.load_source(filename.split('.')[0], commands_path+'/'+filename)
        command.add_parser(subparsers)
    subparsers.required = True
    return subparsers

def main():
    parser = ArgumentParser(
        add_help=False)
    parser.add_argument(
        '--debug',
        action='store_true',
        help='turn on debug mode')
    parser.add_argument(
        '--version',
        choices = VERSIONS,
        const=VERSIONS[0],
        nargs='?',
        help='default=%(const)s; show the version')
    parser.add_argument(
        '--verbose',
        action='store_true',
        help='more verbose output')
    parser.add_argument(
        '--api-url',
        metavar='URL',
        type=URL,
        default=r'http://0.0.0.0',
        help='default=%(default)s; set the api url to use')

    ns, rem = parser.parse_known_args()
    if not any([h in rem for h in ('-h', '--help')]):
        version_check(ns)

    config = {}

    parser = ArgumentParser(
        parents=[parser],
        description=__doc__,
        formatter_class=RawDescriptionHelpFormatter,
        argument_default=config)

    add_subparsers(parser)

    ns = parser.parse_args()
    if ns.debug:
        print('ns =', ns)
    ns.func(ns)
<|endoftext|>


<commit_before>
from sympy import Symbol, simplify, collect, latex, pprint, solve

rho = Symbol("rho")
s = Symbol("s")
p = Symbol("p")
q = Symbol("q")
r = Symbol("r")
eps = Symbol("epsilon")
sigma = Symbol("sigma")

print(collect(simplify( ((p - sigma)**2*(sigma**2 * (s+1) - 2*p*sigma)).expand()
    + r*sigma**2*s**2 - (2*r*s*sigma*(p-sigma)).expand() - ((p - sigma)**2*eps).expand()), sigma))

"""
print(solve(collect(simplify((q*(rho**2)*(s+1)*(rho-1)**2).expand() -
    (2*q*rho*((rho-1)**2) * (rho*(s+1)-s) ).expand() +
    (r*rho**2*(rho*(s+1)-s)**2).expand() -
    (2*r*rho*(rho-1)*(rho*(s+1)-s)**2).expand() -
    (eps*((rho*(s+1)-s)**2) *(rho-1)**2).expand()), rho), rho))"""

"""pprint(collect(simplify((q*(rho**2)*(s+1)*(rho-1)**2).expand() -
    (2*q*rho*((rho-1)**2) * (rho*(s+1)-s) ).expand() +
    (r*rho**2*(rho*(s+1)-s)**2).expand() -
    (2*r*rho*(rho-1)*(rho*(s+1)-s)**2).expand() -
    (eps*((rho*(s+1)-s)**2) *(rho-1)**2).expand()), rho))"""<commit_msg>Rename p->q<commit_after>
from sympy import Symbol, simplify, collect, latex, pprint, solve

rho = Symbol("rho")
s = Symbol("s")
q = Symbol("q")
r = Symbol("r")
eps = Symbol("epsilon")
sigma = Symbol("sigma")

print(collect(simplify( ((q - sigma)**2*(sigma**2 * (s+1) - 2*q*sigma)).expand()
    + r*sigma**2*s**2 - (2*r*s*sigma*(q-sigma)).expand() - ((q - sigma)**2*eps).expand()), sigma))

"""
print(solve(collect(simplify((q*(rho**2)*(s+1)*(rho-1)**2).expand() -
    (2*q*rho*((rho-1)**2) * (rho*(s+1)-s) ).expand() +
    (r*rho**2*(rho*(s+1)-s)**2).expand() -
    (2*r*rho*(rho-1)*(rho*(s+1)-s)**2).expand() -
    (eps*((rho*(s+1)-s)**2) *(rho-1)**2).expand()), rho), rho))"""

"""pprint(collect(simplify((q*(rho**2)*(s+1)*(rho-1)**2).expand() -
    (2*q*rho*((rho-1)**2) * (rho*(s+1)-s) ).expand() +
    (r*rho**2*(rho*(s+1)-s)**2).expand() -
    (2*r*rho*(rho-1)*(rho*(s+1)-s)**2).expand() -
    (eps*((rho*(s+1)-s)**2) *(rho-1)**2).expand()), rho))"""<|endoftext|>