<commit_before># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox
import cybox.bindings.file_object as file_binding
from cybox.common import (DateTime, ExtractedFeatures, HashList, HexBinary,
        ObjectProperties, String, UnsignedLong)

class FilePath(String):
    _binding = file_binding
    _binding_class = file_binding.FilePathType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    def __init__(self, *args, **kwargs):
        String.__init__(self, *args, **kwargs)
        self.fully_qualified = None

    def is_plain(self):
        return (super(FilePath, self).is_plain() and
                self.fully_qualified is None)

    def to_obj(self):
        filepath_obj = String.to_obj(self)
        if self.fully_qualified is not None:
            filepath_obj.set_fully_qualified(self.fully_qualified)
        return filepath_obj

    def to_dict(self):
        filepath_dict = String.to_dict(self)
        if self.fully_qualified is not None:
            filepath_dict['fully_qualified'] = self.fully_qualified
        return filepath_dict

    @staticmethod
    def from_obj(filepath_obj):
        if not filepath_obj:
            return None
        filepath = FilePath()
        filepath._populate_from_obj(filepath_obj)
        filepath.fully_qualified = filepath_obj.get_fully_qualified()
        return filepath

    @staticmethod
    def from_dict(filepath_dict):
        if not filepath_dict:
            return None
        filepath = FilePath()
        filepath._populate_from_dict(filepath_dict)
        if isinstance(filepath_dict, dict):
            filepath.fully_qualified = filepath_dict.get('fully_qualified')
        return filepath

class File(ObjectProperties):
    _binding = file_binding
    _binding_class = file_binding.FileObjectType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
    _XSI_NS = "FileObj"
    _XSI_TYPE = "FileObjectType"

    is_packed = cybox.TypedField("is_packed")
    is_masqueraded = cybox.TypedField("is_masqueraded")
    file_name = cybox.TypedField("File_Name", String)
    file_path = cybox.TypedField("File_Path", FilePath)
    device_path = cybox.TypedField("Device_Path", String)
    full_path = cybox.TypedField("Full_Path", String)
    file_extension = cybox.TypedField("File_Extension", String)
    size_in_bytes = cybox.TypedField("Size_In_Bytes", UnsignedLong)
    magic_number = cybox.TypedField("Magic_Number", HexBinary)
    file_format = cybox.TypedField("File_Format", String)
    modified_time = cybox.TypedField("Modified_Time", String)
    accessed_time = cybox.TypedField("Accessed_Time", String)
    created_time = cybox.TypedField("Created_Time", DateTime)
    hashes = cybox.TypedField("Hashes", HashList)
    extracted_features = cybox.TypedField("Extracted_Features",
                                          ExtractedFeatures)

    # Not supported yet:
    # - Digital_Signatures
    # - File_Attributes_List
    # - Permissions
    # - User_Owner
    # - Packer_List
    # - Peak_Entropy
    # - Sym_Links
    # - Extracted_Features
    # - Byte Runs

    def __init__(self):
        super(File, self).__init__()
        self.is_packed = None

    @property
    def md5(self):
        if self.hashes is None:
            return None
        return self.hashes.md5

    @md5.setter
    def md5(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.md5 = value

    @property
    def sha1(self):
        if self.hashes is None:
            return None
        return self.hashes.sha1

    @sha1.setter
    def sha1(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha1 = value

    @property
    def sha256(self):
        if self.hashes is None:
            return None
        return self.hashes.sha256

    @sha256.setter
    def sha256(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha256 = value

    @property
    def size(self):
        """`size` is an alias for `size_in_bytes`"""
        return self.size_in_bytes

    @size.setter
    def size(self, value):
        """`size` is an alias for `size_in_bytes`"""
        self.size_in_bytes = value

    def add_hash(self, hash_):
        if hash_ is not None:
            if self.hashes is None:
                self.hashes = HashList()
            self.hashes.append(hash_)

class Packer(cybox.Entity):
    _binding = file_binding
    _binding_class = file_binding.PackerType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    name = cybox.TypedField("Name", String)
    version = cybox.TypedField("Version", String)
    entry_point = cybox.TypedField("Entry_Point", HexBinary)
    signature = cybox.TypedField("Signature", String)
    type_ = cybox.TypedField("Type", String)
    #TODO: add Detected_Entrypoint_Signatures and EP_Jump_Codes

class PackerList(cybox.EntityList):
    _binding = file_binding
    _binding_class = file_binding.PackerListType
    _binding_var = "Packer"
    _contained_type = Packer
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
<commit_msg>Added encryption/decryption related properties to File class<commit_after># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox
import cybox.bindings.file_object as file_binding
from cybox.common import (DateTime, ExtractedFeatures, HashList, HexBinary,
        ObjectProperties, String, UnsignedLong)

class FilePath(String):
    _binding = file_binding
    _binding_class = file_binding.FilePathType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    def __init__(self, *args, **kwargs):
        String.__init__(self, *args, **kwargs)
        self.fully_qualified = None

    def is_plain(self):
        return (super(FilePath, self).is_plain() and
                self.fully_qualified is None)

    def to_obj(self):
        filepath_obj = String.to_obj(self)
        if self.fully_qualified is not None:
            filepath_obj.set_fully_qualified(self.fully_qualified)
        return filepath_obj

    def to_dict(self):
        filepath_dict = String.to_dict(self)
        if self.fully_qualified is not None:
            filepath_dict['fully_qualified'] = self.fully_qualified
        return filepath_dict

    @staticmethod
    def from_obj(filepath_obj):
        if not filepath_obj:
            return None
        filepath = FilePath()
        filepath._populate_from_obj(filepath_obj)
        filepath.fully_qualified = filepath_obj.get_fully_qualified()
        return filepath

    @staticmethod
    def from_dict(filepath_dict):
        if not filepath_dict:
            return None
        filepath = FilePath()
        filepath._populate_from_dict(filepath_dict)
        if isinstance(filepath_dict, dict):
            filepath.fully_qualified = filepath_dict.get('fully_qualified')
        return filepath

class File(ObjectProperties):
    _binding = file_binding
    _binding_class = file_binding.FileObjectType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
    _XSI_NS = "FileObj"
    _XSI_TYPE = "FileObjectType"

    is_packed = cybox.TypedField("is_packed")
    is_masqueraded = cybox.TypedField("is_masqueraded")
    file_name = cybox.TypedField("File_Name", String)
    file_path = cybox.TypedField("File_Path", FilePath)
    device_path = cybox.TypedField("Device_Path", String)
    full_path = cybox.TypedField("Full_Path", String)
    file_extension = cybox.TypedField("File_Extension", String)
    size_in_bytes = cybox.TypedField("Size_In_Bytes", UnsignedLong)
    magic_number = cybox.TypedField("Magic_Number", HexBinary)
    file_format = cybox.TypedField("File_Format", String)
    modified_time = cybox.TypedField("Modified_Time", String)
    accessed_time = cybox.TypedField("Accessed_Time", String)
    created_time = cybox.TypedField("Created_Time", DateTime)
    hashes = cybox.TypedField("Hashes", HashList)
    extracted_features = cybox.TypedField("Extracted_Features",
                                          ExtractedFeatures)
    encryption_algorithm = cybox.TypedField("Encryption_Algorithm", String)
    decryption_key = cybox.TypedField("Decryption_Key", String)
    compression_method = cybox.TypedField("Compression_Method", String)
    compression_version = cybox.TypedField("Compression_Version", String)
    compression_comment = cybox.TypedField("Compression_Comment", String)

    # Not supported yet:
    # - Digital_Signatures
    # - File_Attributes_List
    # - Permissions
    # - User_Owner
    # - Packer_List
    # - Peak_Entropy
    # - Sym_Links
    # - Extracted_Features
    # - Byte Runs

    def __init__(self):
        super(File, self).__init__()
        self.is_packed = None

    @property
    def md5(self):
        if self.hashes is None:
            return None
        return self.hashes.md5

    @md5.setter
    def md5(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.md5 = value

    @property
    def sha1(self):
        if self.hashes is None:
            return None
        return self.hashes.sha1

    @sha1.setter
    def sha1(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha1 = value

    @property
    def sha256(self):
        if self.hashes is None:
            return None
        return self.hashes.sha256

    @sha256.setter
    def sha256(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha256 = value

    @property
    def size(self):
        """`size` is an alias for `size_in_bytes`"""
        return self.size_in_bytes

    @size.setter
    def size(self, value):
        """`size` is an alias for `size_in_bytes`"""
        self.size_in_bytes = value

    def add_hash(self, hash_):
        if hash_ is not None:
            if self.hashes is None:
                self.hashes = HashList()
            self.hashes.append(hash_)

class Packer(cybox.Entity):
    _binding = file_binding
    _binding_class = file_binding.PackerType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    name = cybox.TypedField("Name", String)
    version = cybox.TypedField("Version", String)
    entry_point = cybox.TypedField("Entry_Point", HexBinary)
    signature = cybox.TypedField("Signature", String)
    type_ = cybox.TypedField("Type", String)
    #TODO: add Detected_Entrypoint_Signatures and EP_Jump_Codes

class PackerList(cybox.EntityList):
    _binding = file_binding
    _binding_class = file_binding.PackerListType
    _binding_var = "Packer"
    _contained_type = Packer
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
<|endoftext|>


<commit_before>from django.core.exceptions import ValidationError
from django.db import models
from django.http import HttpResponse

from cyder.base.constants import IP_TYPES, IP_TYPE_4, IP_TYPE_6
from cyder.base.mixins import ObjectUrlMixin
from cyder.base.helpers import get_display
from cyder.cydhcp.constants import (
    ALLOW_OPTIONS, DENY_OPTIONS, RANGE_TYPE, STATIC
)
from cyder.cydns.validation import validate_ip_type
from cyder.cydhcp.constants import (ALLOW_VRF, ALLOW_KNOWN,
                                    ALLOW_LEGACY)
from cyder.cydhcp.interface.static_intr.models import StaticInterface
from cyder.cydhcp.network.models import Network
from cyder.cydhcp.utils import IPFilter, four_to_two, join_dhcp_args
from cyder.cydhcp.keyvalue.utils import AuxAttr
from cyder.cydhcp.keyvalue.base_option import CommonOption
from cyder.cydns.address_record.models import AddressRecord
from cyder.cydns.ip.models import ipv6_to_longs
from cyder.cydns.ptr.models import PTR

import ipaddr
# import reversion

class Range(models.Model, ObjectUrlMixin):
    """The Range class.

        >>> Range(start=start_ip, end=end_ip,
        >>>         defualt_domain=domain, network=network)

    Ranges live inside networks; their start ip address is greater than or
    equal to the the start of their network and their end ip address is less
    than or equal to the end of their network; both the Range and the network
    class enforce these requirements. Good practice says ranges should not
    start on the network address of their network and they should not end on
    the broadcast address of their network; the Range and Network classes do
    not enforce this.

    Changing a Range

    Things that happen when a static range is changed:

        *   The new `start` and `end` values are checked against the range's
            network to ensure that the range still exists within the network.
        *   The new `start` and `end` values are checked against all other
            existing range's `start` and `end` values to make sure that the new
            range does not overlap.
    """

    id = models.AutoField(primary_key=True)
    network = models.ForeignKey(Network, null=True, blank=True)

    ip_type = models.CharField(
        verbose_name='IP address type', max_length=1,
        choices=IP_TYPES.items(), default=IP_TYPE_4,
        validators=[validate_ip_type]
    )
    start_upper = models.BigIntegerField(null=True)
    start_lower = models.BigIntegerField(null=True)
    start_str = models.CharField(max_length=39, editable=True)

    end_lower = models.BigIntegerField(null=True)
    end_upper = models.BigIntegerField(null=True)
    end_str = models.CharField(max_length=39, editable=True)

    is_reserved = models.BooleanField(default=False, blank=False)

    allow = models.CharField(max_length=1, choices=ALLOW_OPTIONS,
                             default=ALLOW_LEGACY)
    deny = models.CharField(max_length=20, choices=DENY_OPTIONS.items(),
                            blank=True)

    attrs = None
    dhcpd_raw_include = models.TextField(blank=True)
    dhcp_enabled = models.BooleanField(default=True)
    range_type = models.CharField(max_length=2, choices=RANGE_TYPE.items(),
                                  default=STATIC, editable=False)

    search_fields = ('start_str', 'end_str')
    display_fields = ('start_str', 'end_str')

    class Meta:
        db_table = 'range'
        unique_together = ('start_upper', 'start_lower', 'end_upper',
                           'end_lower')

    def __str__(self):
        return get_display(self)

    def __repr__(self):
        return "<Range: {0}>".format(str(self))

    @staticmethod
    def filter_by_ctnr(ctnr, objects=None):
        if objects:
            return ctnr.ranges.filter(pk__in=objects)
        else:
            return ctnr.ranges

    def update_attrs(self):
        self.attrs = AuxAttr(RangeKeyValue, self, "range")

    def _range_ips(self):
        self._start, self._end = four_to_two(
            self.start_upper,
            self.start_lower,
            self.end_upper,
            self.end_lower)

    def details(self):
        """For tables."""
        data = super(Range, self).details()
        has_net = self.network is not None
        data['data'] = [
            ('Range', 'start_str', self),
            ('Network', 'network', self.network if has_net else ""),
            ('Site', 'network__site', self.network.site if has_net else ""),
            ('Vlan', 'network__vlan', self.network.vlan if has_net else "")]
        return data

    @staticmethod
    def eg_metadata():
        """EditableGrid metadata."""
        return {'metadata': [
            {'name': 'start_str', 'datatype': 'string', 'editable': False},
            {'name': 'network', 'datatype': 'string', 'editable': False},
            {'name': 'site', 'datatype': 'string', 'editable': False},
            {'name': 'vlan', 'datatype': 'string', 'editable': False},
        ]}

    def save(self, *args, **kwargs):
        self.clean()
        super(Range, self).save(*args, **kwargs)

    def clean(self):
        if self.network is None and not self.is_reserved:
            raise ValidationError("ERROR: Range {0}-{1} is not associated "
                                  "with a network and is not reserved".format(
                                  self.start_str, self.end_str))
        try:
            if self.ip_type == IP_TYPE_4:
                self.start_upper, self.start_lower = 0, int(
                    ipaddr.IPv4Address(self.start_str))
                self.end_upper, self.end_lower = 0, int(
                    ipaddr.IPv4Address(self.end_str))
            elif self.ip_type == IP_TYPE_6:
                self.start_upper, self.start_lower = ipv6_to_longs(
                    self.start_str)
                self.end_upper, self.end_lower = ipv6_to_longs(self.end_str)
            else:
                raise ValidationError("ERROR: could not determine the ip type")
        except ipaddr.AddressValueError, e:
            raise ValidationError(str(e))

        """
        Some notes:
        start = s1 s2
        end = e1 e2

        if s1 > e1:
            start > end
            # Bad
        if e1 > s1:
            end > start
            # Good
        if s1 == e1 and s2 > e2:
            start > end
            # Bad
        if s1 == e1 and s2 < e2:
            end > start
            # Good
        if s1 == e1 and s2 == e2:
            end == start
            # Bad
        """
        start, end = four_to_two(
            self.start_upper, self.start_lower, self.end_upper, self.end_lower)
        if start > end:
            raise ValidationError("The start of a range cannot be greater than"
                                  " or equal to the end of the range.")
        if not self.is_reserved:
            self.network.update_network()
            if self.network.ip_type == IP_TYPE_4:
                IPClass = ipaddr.IPv4Address
            else:
                IPClass = ipaddr.IPv6Address

            if (IPClass(self.start_str) < self.network.network.network or
                    IPClass(self.end_str) > self.network.network.broadcast):
                raise RangeOverflowError(
                    "Range {0} to {1} doesn't fit in {2}".format(
                        IPClass(self.start_lower),
                        IPClass(self.end_lower),
                        self.network.network))
        self.check_for_overlaps()

    def get_allowed_clients(self):
        allow = []
        if self.allow == ALLOW_VRF:
            allow = ["allow members of \"{0}:{1}:{2}\"".format(
                self.network.vrf.name, self.start_str, self.end_str)]
        elif self.allow == ALLOW_KNOWN:
            allow = ['allow known clients']
        elif self.allow == ALLOW_LEGACY:
            allow = ["allow members of \"{0}:{1}:{2}\"".format(
                ctnr.name, self.start_str, self.end_str)
                for ctnr in self.ctnr_set.all()]
        return allow

    def check_for_overlaps(self):
        """
        This function will look at all the other ranges and make sure we don't
        overlap with any of them.
        """
        self._range_ips()
        if self.ip_type == IP_TYPE_4:
            Ip = ipaddr.IPv4Address
        else:
            Ip = ipaddr.IPv6Address

        for range in Range.objects.all():
            if range.pk == self.pk:
                continue
            range._range_ips()
            #the range being tested is above this range
            if self._start > range._end:
                continue
            # start > end
            if self._end < range._start:
                continue
            raise ValidationError("Stored range {0} - {1} would contain "
                                  "{2} - {3}".format(
                                      Ip(range._start),
                                      Ip(range._end),
                                      Ip(self._start),
                                      Ip(self._end)))

    def build_range(self):
        range_options = self.rangekeyvalue_set.filter(is_option=True)
        range_statements = self.rangekeyvalue_set.filter(is_statement=True)
        build_str = "\tpool {\n"
        build_str += "\t\t# Pool Statements\n"
        build_str += "\t\tfailover peer \"dhcp\";\n"
        build_str += "\t\tdeny dynamic bootp clients;\n"
        build_str += join_dhcp_args(range_statements, depth=2)
        if range_options:
            build_str += "\t\t# Pool Options\n"
            build_str += join_dhcp_args(range_options, depth=2)
        if self.dhcpd_raw_include:
            build_str += "\t\t# Raw pool includes\n"
            build_str += "\t\t{0};".format(self.dhcp_raw_include)
        build_str += "\t\t# Allow statements\n"
        build_str += join_dhcp_args(self.get_allowed_clients(), depth=2)
        if self.ip_type == IP_TYPE_4:
            build_str += "\t\trange {0} {1};\n".format(self.start_str,
                                                       self.end_str)
        else:
            build_str += "\t\trange6{0} {1};\n".format(self.start_str,
                                                       self.end_str)
        build_str += "\t}\n\n"
        return build_str

    def update_ipf(self):
        """Update the IP filter. Used for compiling search queries and firewall
        rules."""
        self.ipf = IPFilter(self.start_str, self.end_str, self.network.ip_type,
                            object_=self)

    def display(self):
        return "Range: {3} to {4} {0} -- {2} -- {1}".format(
            self.network.site, self.network.vlan, self.network,
            self.start_str, self.end_str)

    def choice_display(self):
        if not self.network.site:
            site_name = "No Site"
        else:
            site_name = self.network.site.name.upper()

        if not self.network.vlan:
            vlan_name = "No Vlan"
        else:
            vlan_name = str(self.network.vlan)
        return "{0} - {1} - ({2}) {3} to {4}".format(
            site_name, vlan_name,
            self.network, self.start_str, self.end_str)

    def get_next_ip(self):
        """Find's the most appropriate ip address within a range. If it can't
        find an IP it returns None. If it finds an IP it returns an IPv4Address
        object.

            :returns: ipaddr.IPv4Address
        """
        if self.network.ip_type is not '4':
            return None
        start = self.start_lower
        end = self.end_lower
        if start >= end - 1:
            return HttpResponse("Too small of a range.")
        ip = find_free_ip(start, end, ip_type='4')
        if ip:
            return ip
        else:
            return None

def find_free_ip(start, end, ip_type='4'):
    """Given start and end numbers, find a free ip.
    :param start: The start number
    :type start: int
    :param end: The end number
    :type end: int
    :param ip_type: The type of IP you are looking for.
    :type ip_type: str either '4' or '6'
    """
    if ip_type == IP_TYPE_4:
        records = AddressRecord.objects.filter(ip_upper=0, ip_lower__gte=start,
                                               ip_lower__lte=end)
        ptrs = PTR.objects.filter(ip_upper=0, ip_lower__gte=start,
                                  ip_lower__lte=end)
        intrs = StaticInterface.objects.filter(ip_upper=0, ip_lower__gte=start,
                                               ip_lower__lte=end)
        if not records and not intrs:
            ip = ipaddr.IPv4Address(start)
            return ip
        for i in xrange(start, end + 1):
            taken = False
            for record in records:
                if record.ip_lower == i:
                    taken = True
                    break
            for ptr in ptrs:
                if ptr.ip_lower == i:
                    taken = True
                    break
            if not taken:
                for intr in intrs:
                    if intr.ip_lower == i:
                        taken = True
                        break
            if not taken:
                ip = ipaddr.IPv4Address(i)
                return ip
    else:
        raise NotImplemented

# reversion.(Range)

class RangeKeyValue(CommonOption):
    range = models.ForeignKey(Range, null=False)

    class Meta:
        db_table = 'range_kv'
        unique_together = ('key', 'value', 'range')

    def _aa_failover(self):
        self.is_statement = True
        self.is_option = False
        if self.value != "peer \"dhcp-failover\"":
            raise ValidationError("Invalid failover option. Try `peer "
                                  "\"dhcp-failover\"`")

    def _aa_routers(self):
        self._routers(self.range.network.ip_type)

    def _aa_ntp_servers(self):
        self._ntp_servers(self.range.network.ip_type)

# reversion.(RangeKeyValue)

class RangeOverflowError(ValidationError):
    pass
<commit_msg>Make range_type editable; validate interfaces<commit_after>from django.core.exceptions import ValidationError
from django.db import models
from django.http import HttpResponse

from cyder.base.constants import IP_TYPES, IP_TYPE_4, IP_TYPE_6
from cyder.base.mixins import ObjectUrlMixin
from cyder.base.helpers import get_display
from cyder.cydhcp.constants import (
    ALLOW_OPTIONS, DENY_OPTIONS, RANGE_TYPE, STATIC
)
from cyder.cydns.validation import validate_ip_type
from cyder.cydhcp.constants import (ALLOW_VRF, ALLOW_KNOWN,
                                    ALLOW_LEGACY)
from cyder.cydhcp.interface.static_intr.models import StaticInterface
from cyder.cydhcp.network.models import Network
from cyder.cydhcp.utils import (IPFilter, four_to_two, join_dhcp_args,
                                start_end_filter)
from cyder.cydhcp.keyvalue.utils import AuxAttr
from cyder.cydhcp.keyvalue.base_option import CommonOption
from cyder.cydns.address_record.models import AddressRecord
from cyder.cydns.ip.models import ipv6_to_longs
from cyder.cydns.ptr.models import PTR

import ipaddr
# import reversion

class Range(models.Model, ObjectUrlMixin):
    """The Range class.

        >>> Range(start=start_ip, end=end_ip,
        >>>         defualt_domain=domain, network=network)

    Ranges live inside networks; their start ip address is greater than or
    equal to the the start of their network and their end ip address is less
    than or equal to the end of their network; both the Range and the network
    class enforce these requirements. Good practice says ranges should not
    start on the network address of their network and they should not end on
    the broadcast address of their network; the Range and Network classes do
    not enforce this.

    Changing a Range

    Things that happen when a static range is changed:

        *   The new `start` and `end` values are checked against the range's
            network to ensure that the range still exists within the network.
        *   The new `start` and `end` values are checked against all other
            existing range's `start` and `end` values to make sure that the new
            range does not overlap.
    """

    id = models.AutoField(primary_key=True)
    network = models.ForeignKey(Network, null=True, blank=True)

    ip_type = models.CharField(
        verbose_name='IP address type', max_length=1,
        choices=IP_TYPES.items(), default=IP_TYPE_4,
        validators=[validate_ip_type]
    )
    start_upper = models.BigIntegerField(null=True)
    start_lower = models.BigIntegerField(null=True)
    start_str = models.CharField(max_length=39, editable=True)

    end_lower = models.BigIntegerField(null=True)
    end_upper = models.BigIntegerField(null=True)
    end_str = models.CharField(max_length=39, editable=True)

    is_reserved = models.BooleanField(default=False, blank=False)

    allow = models.CharField(max_length=1, choices=ALLOW_OPTIONS,
                             default=ALLOW_LEGACY)
    deny = models.CharField(max_length=20, choices=DENY_OPTIONS.items(),
                            blank=True)

    attrs = None
    dhcpd_raw_include = models.TextField(blank=True)
    dhcp_enabled = models.BooleanField(default=True)
    range_type = models.CharField(max_length=2, choices=RANGE_TYPE.items(),
                                  default=STATIC)

    search_fields = ('start_str', 'end_str')
    display_fields = ('start_str', 'end_str')

    class Meta:
        db_table = 'range'
        unique_together = ('start_upper', 'start_lower', 'end_upper',
                           'end_lower')

    def __str__(self):
        return get_display(self)

    def __repr__(self):
        return "<Range: {0}>".format(str(self))

    @staticmethod
    def filter_by_ctnr(ctnr, objects=None):
        if objects:
            return ctnr.ranges.filter(pk__in=objects)
        else:
            return ctnr.ranges

    def update_attrs(self):
        self.attrs = AuxAttr(RangeKeyValue, self, "range")

    def _range_ips(self):
        self._start, self._end = four_to_two(
            self.start_upper,
            self.start_lower,
            self.end_upper,
            self.end_lower)

    def details(self):
        """For tables."""
        data = super(Range, self).details()
        has_net = self.network is not None
        data['data'] = [
            ('Range', 'start_str', self),
            ('Network', 'network', self.network if has_net else ""),
            ('Site', 'network__site', self.network.site if has_net else ""),
            ('Vlan', 'network__vlan', self.network.vlan if has_net else "")]
        return data

    @staticmethod
    def eg_metadata():
        """EditableGrid metadata."""
        return {'metadata': [
            {'name': 'start_str', 'datatype': 'string', 'editable': False},
            {'name': 'network', 'datatype': 'string', 'editable': False},
            {'name': 'site', 'datatype': 'string', 'editable': False},
            {'name': 'vlan', 'datatype': 'string', 'editable': False},
        ]}

    def save(self, *args, **kwargs):
        self.clean()
        super(Range, self).save(*args, **kwargs)

    def clean(self):
        if self.network is None and not self.is_reserved:
            raise ValidationError("ERROR: Range {0}-{1} is not associated "
                                  "with a network and is not reserved".format(
                                  self.start_str, self.end_str))
        try:
            if self.ip_type == IP_TYPE_4:
                self.start_upper, self.start_lower = 0, int(
                    ipaddr.IPv4Address(self.start_str))
                self.end_upper, self.end_lower = 0, int(
                    ipaddr.IPv4Address(self.end_str))
            elif self.ip_type == IP_TYPE_6:
                self.start_upper, self.start_lower = ipv6_to_longs(
                    self.start_str)
                self.end_upper, self.end_lower = ipv6_to_longs(self.end_str)
            else:
                raise ValidationError("ERROR: could not determine the ip type")
        except ipaddr.AddressValueError, e:
            raise ValidationError(str(e))

        """
        Some notes:
        start = s1 s2
        end = e1 e2

        if s1 > e1:
            start > end
            # Bad
        if e1 > s1:
            end > start
            # Good
        if s1 == e1 and s2 > e2:
            start > end
            # Bad
        if s1 == e1 and s2 < e2:
            end > start
            # Good
        if s1 == e1 and s2 == e2:
            end == start
            # Bad
        """
        start, end = four_to_two(
            self.start_upper, self.start_lower, self.end_upper, self.end_lower)
        if start > end:
            raise ValidationError("The start of a range cannot be greater than"
                                  " or equal to the end of the range.")

        if self.range_type == STATIC:
            if self.dynamicinterface_set.exists():
                raise ValidationError('A dynamic range cannot contain static '
                                      'interfaces')
        else:
            if StaticInterface.objects.filter(
                    start_end_filter(start, end, self.ip_type)[2]).exists():
                raise ValidationError('A static range cannot contain dynamic '
                                      'interfaces')

        if not self.is_reserved:
            self.network.update_network()
            if self.network.ip_type == IP_TYPE_4:
                IPClass = ipaddr.IPv4Address
            else:
                IPClass = ipaddr.IPv6Address

            if (IPClass(self.start_str) < self.network.network.network or
                    IPClass(self.end_str) > self.network.network.broadcast):
                raise RangeOverflowError(
                    "Range {0} to {1} doesn't fit in {2}".format(
                        IPClass(self.start_lower),
                        IPClass(self.end_lower),
                        self.network.network))
        self.check_for_overlaps()

    def get_allowed_clients(self):
        allow = []
        if self.allow == ALLOW_VRF:
            allow = ["allow members of \"{0}:{1}:{2}\"".format(
                self.network.vrf.name, self.start_str, self.end_str)]
        elif self.allow == ALLOW_KNOWN:
            allow = ['allow known clients']
        elif self.allow == ALLOW_LEGACY:
            allow = ["allow members of \"{0}:{1}:{2}\"".format(
                ctnr.name, self.start_str, self.end_str)
                for ctnr in self.ctnr_set.all()]
        return allow

    def check_for_overlaps(self):
        """
        This function will look at all the other ranges and make sure we don't
        overlap with any of them.
        """
        self._range_ips()
        if self.ip_type == IP_TYPE_4:
            Ip = ipaddr.IPv4Address
        else:
            Ip = ipaddr.IPv6Address

        for range in Range.objects.all():
            if range.pk == self.pk:
                continue
            range._range_ips()
            #the range being tested is above this range
            if self._start > range._end:
                continue
            # start > end
            if self._end < range._start:
                continue
            raise ValidationError("Stored range {0} - {1} would contain "
                                  "{2} - {3}".format(
                                      Ip(range._start),
                                      Ip(range._end),
                                      Ip(self._start),
                                      Ip(self._end)))

    def build_range(self):
        range_options = self.rangekeyvalue_set.filter(is_option=True)
        range_statements = self.rangekeyvalue_set.filter(is_statement=True)
        build_str = "\tpool {\n"
        build_str += "\t\t# Pool Statements\n"
        build_str += "\t\tfailover peer \"dhcp\";\n"
        build_str += "\t\tdeny dynamic bootp clients;\n"
        build_str += join_dhcp_args(range_statements, depth=2)
        if range_options:
            build_str += "\t\t# Pool Options\n"
            build_str += join_dhcp_args(range_options, depth=2)
        if self.dhcpd_raw_include:
            build_str += "\t\t# Raw pool includes\n"
            build_str += "\t\t{0};".format(self.dhcp_raw_include)
        build_str += "\t\t# Allow statements\n"
        build_str += join_dhcp_args(self.get_allowed_clients(), depth=2)
        if self.ip_type == IP_TYPE_4:
            build_str += "\t\trange {0} {1};\n".format(self.start_str,
                                                       self.end_str)
        else:
            build_str += "\t\trange6{0} {1};\n".format(self.start_str,
                                                       self.end_str)
        build_str += "\t}\n\n"
        return build_str

    def update_ipf(self):
        """Update the IP filter. Used for compiling search queries and firewall
        rules."""
        self.ipf = IPFilter(self.start_str, self.end_str, self.network.ip_type,
                            object_=self)

    def display(self):
        return "Range: {3} to {4} {0} -- {2} -- {1}".format(
            self.network.site, self.network.vlan, self.network,
            self.start_str, self.end_str)

    def choice_display(self):
        if not self.network.site:
            site_name = "No Site"
        else:
            site_name = self.network.site.name.upper()

        if not self.network.vlan:
            vlan_name = "No Vlan"
        else:
            vlan_name = str(self.network.vlan)
        return "{0} - {1} - ({2}) {3} to {4}".format(
            site_name, vlan_name,
            self.network, self.start_str, self.end_str)

    def get_next_ip(self):
        """Find's the most appropriate ip address within a range. If it can't
        find an IP it returns None. If it finds an IP it returns an IPv4Address
        object.

            :returns: ipaddr.IPv4Address
        """
        if self.network.ip_type is not '4':
            return None
        start = self.start_lower
        end = self.end_lower
        if start >= end - 1:
            return HttpResponse("Too small of a range.")
        ip = find_free_ip(start, end, ip_type='4')
        if ip:
            return ip
        else:
            return None

def find_free_ip(start, end, ip_type='4'):
    """Given start and end numbers, find a free ip.
    :param start: The start number
    :type start: int
    :param end: The end number
    :type end: int
    :param ip_type: The type of IP you are looking for.
    :type ip_type: str either '4' or '6'
    """
    if ip_type == IP_TYPE_4:
        records = AddressRecord.objects.filter(ip_upper=0, ip_lower__gte=start,
                                               ip_lower__lte=end)
        ptrs = PTR.objects.filter(ip_upper=0, ip_lower__gte=start,
                                  ip_lower__lte=end)
        intrs = StaticInterface.objects.filter(ip_upper=0, ip_lower__gte=start,
                                               ip_lower__lte=end)
        if not records and not intrs:
            ip = ipaddr.IPv4Address(start)
            return ip
        for i in xrange(start, end + 1):
            taken = False
            for record in records:
                if record.ip_lower == i:
                    taken = True
                    break
            for ptr in ptrs:
                if ptr.ip_lower == i:
                    taken = True
                    break
            if not taken:
                for intr in intrs:
                    if intr.ip_lower == i:
                        taken = True
                        break
            if not taken:
                ip = ipaddr.IPv4Address(i)
                return ip
    else:
        raise NotImplemented

# reversion.(Range)

class RangeKeyValue(CommonOption):
    range = models.ForeignKey(Range, null=False)

    class Meta:
        db_table = 'range_kv'
        unique_together = ('key', 'value', 'range')

    def _aa_failover(self):
        self.is_statement = True
        self.is_option = False
        if self.value != "peer \"dhcp-failover\"":
            raise ValidationError("Invalid failover option. Try `peer "
                                  "\"dhcp-failover\"`")

    def _aa_routers(self):
        self._routers(self.range.network.ip_type)

    def _aa_ntp_servers(self):
        self._ntp_servers(self.range.network.ip_type)

# reversion.(RangeKeyValue)

class RangeOverflowError(ValidationError):
    pass
<|endoftext|>


<commit_before>import os.path
import os
import hashlib

from dogpile.cache.api import CacheBackend, NO_VALUE
from dogpile.cache.api import CachedValue

class CacheFileBackend(CacheBackend):
    def __init__(self, arguments):
        self._root = arguments.get('root', '/tmp/cache')
        self._cache = {}

        if not os.path.isdir(self._root):
            # todo intermediaire
            os.mkdir(self._root)

    @staticmethod
    def _computeHash(key):
        return hashlib.md5(key).hexdigest()

    def get(self, key):
        cacheHash = self._computeHash(key)

        path = os.path.join(self._root, cacheHash)

        if not os.path.isfile(path):
            return NO_VALUE

        with open(path, 'rb') as tmp:
            metadata = self._cache[cacheHash]

            return CachedValue(tmp.read(), metadata)

    def set(self, key, value):
        cacheHash = self._computeHash(key)
        path = os.path.join(self._root, cacheHash)

        # import rpdb
        # rpdb.set_trace()

        with open(path, 'wb') as tmp:
            tmp.write(value.payload)

        self._cache[cacheHash] = value.metadata

    def delete(self, key):
        cacheHash = self._computeHash(key)
        path = os.path.join(self._root, cacheHash)
        os.remove(path)
<commit_msg>storing meta in json file<commit_after>import os.path
import os
import hashlib
import json

from dogpile.cache.api import CacheBackend, NO_VALUE
from dogpile.cache.api import CachedValue

class CacheFileBackend(CacheBackend):
    def __init__(self, arguments):
        self._root = arguments.get('root', '/tmp/cache')
        # self._cache = {}

        if not os.path.isdir(self._root):
            # todo intermediaire
            os.mkdir(self._root)

    @staticmethod
    def _computeHash(key):
        return hashlib.md5(key).hexdigest()

    def get(self, key):
        cacheHash = self._computeHash(key)

        path = os.path.join(self._root, cacheHash)
        pathJson = os.path.join(self._root, cacheHash+'.json')

        if not os.path.isfile(path) or not os.path.isfile(pathJson):
            return NO_VALUE

        metadata = {}

        with open(pathJson, 'rb') as tmp:
            metadata = json.loads(tmp.read())

        with open(path, 'rb') as tmp:
            return CachedValue(tmp.read(), metadata)

    def set(self, key, value):
        cacheHash = self._computeHash(key)
        path = os.path.join(self._root, cacheHash)
        pathJson = os.path.join(self._root, cacheHash+'.json')
        # import rpdb
        # rpdb.set_trace()

        with open(path, 'wb') as tmp:
            tmp.write(value.payload)

        with open(pathJson, 'wb') as tmp:
            tmp.write(json.dumps(value.metadata))

    def delete(self, key):
        cacheHash = self._computeHash(key)
        path = os.path.join(self._root, cacheHash)
        pathJson = os.path.join(self._root, cacheHash+'.json')

        os.remove(path)
        os.remove(pathJson)
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
import pytz

from dace import _

DEFAULT_LOCK_DURATION = timedelta(seconds=120)

class AlreadyLocked(Exception):
    pass

class LockableElement(object):
    _lock = (None, None)
    _lock_duration = DEFAULT_LOCK_DURATION

    def lock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        if self.is_locked(request):
            raise AlreadyLocked(_(u"Already locked by ${user} at ${datetime}",
                mapping={'user': self._lock[0], 'datetime': self._lock[1]}))
        self._lock = (request.principal.id, datetime.now(pytz.utc))

    def unlock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        if self.is_locked(request):
            raise AlreadyLocked(_(u"Already locked by ${user} at ${datetime}",
                mapping={'user': self._lock[0], 'datetime': self._lock[1]}))
        self._lock = (None, None)

    def is_locked(self, request):
        """If the activity was locked by the same user, return False.
        """
        if self._lock[1] is None:
            return False
        if self._lock[1] + self._lock_duration <= datetime.now(pytz.utc):
            return False
        if self._lock[0] == request.principal.id:
            return False
        return True

<commit_msg>use substanced lock library<commit_after>from substanced.locking import(
    lock_resource,
    unlock_resource,
    could_lock_resource,
    LockError,
    UnlockError)

DEFAUT_DURATION = 10

class LockableElement(object):

    def lock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        try:
            lock_resource(self, request.user, DEFAUT_DURATION)
        except LockError:
            return

    def unlock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        try:
            unlock_resource(self, request.user)
        except UnlockError:
            return

    def is_locked(self, request):
        """If the activity was locked by the same user, return False.
        """
        try:
            return not could_lock_resource(self, request.user)
        except LockError:
            return True
<|endoftext|>


<commit_before>from UserString import UserString

RE_CASE = "CASE WHEN ({0}->'{1}' ~ E'"
RE_THEN = "') THEN "
RE_ELSE = "ELSE -2147483648 END"

class ExpressionStr(UserString):
    """
    Subclass of str() that allows annotations based on which key and
    test are being used.
    """
    daff_key = ''
    daff_test = ''
    daff_val = ''

def breaks_optimizer(expr):
    if not isinstance(expr, ExpressionStr):
        return True

    return (
        expr.daff_test in {"!=", "!in"} or
        (expr.daff_test == "?=" and expr.daff_val == "false") or
        (expr.daff_test == "=" and isinstance(expr.daff_val, basestring))
    )

def escape_string_sql(s):
    return "'{}'".format(s)

def make_sql_array(*strings):
    return "ARRAY[{}]".format(
        ",".join(escape_string_sql(s) for s in strings)
    )

class HStoreQueryDelegate(object):

    def __init__(self, hstore_field_name):
        self.field = hstore_field_name

    def mk_any(self, children):
        if children == []:
            return "false"

        sql_expr = " OR ".join("({})".format(child_exp) for child_exp in children)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?| {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " OR ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND ({1})".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_all(self, children):
        if children == ['']:
            return "true"

        sql_expr = " AND ".join("({})".format(child_exp) for child_exp in children if child_exp)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?& {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " AND ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND {1}".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_not_any(self, children):
        return " NOT ({0})".format(self.mk_any(children))

    def mk_not_all(self, children):
        return " NOT ({0})".format(self.mk_all(children))

    def mk_test(self, daff_test_str):
        test_str = daff_test_str

        if test_str == "?=":
            test_fn = lambda k, v, t: "{0}?{1}".format(k, v)
            test_fn.is_datapoint_test = True
        else:
            test_fn = lambda k, v, t: "{0} {1} {2}".format(k, test_str, v)
            if test_str == "!=":
                test_fn.is_NE_test = True
            elif test_str == "=":
                test_fn = lambda k, v, t: "({0} {1}))".format(k, v) if t == basestring else "{0} {1} {2}".format(k, test_str, v)
                test_fn.is_EQ_test = True
            elif test_str == "!in":
                test_fn.is_NOT_IN_test = True
                test_str = "NOT IN"
            elif test_str == "in":
                test_fn.is_IN_test = True

        test_fn.test_str = daff_test_str
        return test_fn

    def mk_cmp(self, key, val, test):
        daff_key = key
        daff_test = test.test_str
        daff_val = val

        if getattr(test, "is_datapoint_test", False):
            # here we cover:
            # [NOT] hstore_col ? '1ukmoviestudios - Disney'
            negate = "NOT " if val == False else ""
            val = "'{0}'".format(key)
            key = "{0}{1}".format(negate, self.field)
            _type = None
        else:
            is_eq_test = getattr(test, "is_EQ_test", False)
            _type, cast, val, type_check = self.cond_cast(val)

            if getattr(test, "is_NE_test", False) or getattr(test, "is_NOT_IN_test", False):
                # here we cover:
                # NOT (hstore_col?'wrong attribute') OR (hstore_col->'wrong attribute')::integer != 2
                key_format = "NOT ({0}?'{1}') OR %s {3} ({0}->'{1}'){2}"
                # if its cast - exclude those not matching type
                key_format = key_format % (" NOT " + type_check[0] + " OR " if cast else "")

            elif is_eq_test and _type == basestring:
                # here we convert '=' to '@>'
                # instead of:
                #   (hs_data->'univisionlanguage1') = 'both'
                # we do:
                #   hs_data @> hstore('univisionlanguage1', 'both')
                key_format = "{0} @> hstore('{1}',"

            elif getattr(test, "is_IN_test", False) or is_eq_test:
                # here we convert '=' to '? AND =':
                # hs_answers?'industries - luxury' AND hs_answers->'industries - luxury' = 'yes'
                key_format = "({0}?'{1}') AND {3} ({0}->'{1}'){2}"

            else:
                key_format = "{3} ({0}->'{1}'){2} "

            key = key_format.format(self.field, key, cast, "".join(type_check)).format(self.field, key)

        expr = ExpressionStr(test(key, val, _type))
        expr.daff_key = daff_key
        expr.daff_test = daff_test
        expr.daff_val = daff_val

        return expr

    def cond_cast(self, val):
        def escape_single_quote(val):
            if isinstance(val, basestring):
                return val.replace(u"'", u"''")
            return val

        def format_list(lst):
            delimiter = "'" if isinstance(lst[0], basestring) else ""
            formatted_list = ",".join(
                [u"{1}{0}{1}".format(escape_single_quote(elem), delimiter) for elem in lst]
            )
            return u"({0})".format(formatted_list)

        def get_cast_attr(val, attr=None):
            for m in CAST_AND_TYPE_MAP:
                if isinstance(val, m["type"]):

                    if attr:
                        return m[attr](val)

                    return [m["type"]] + [m[a](val) for a in ["cast", "value", "type_check"]]

        NUMERIC_TYPE_CHECK = [
                "({0}->'{1}') ~ E'^(?=.+)(?:[1-9]\\\d*|0)?(?:\\\.\\\d+)?$'",
                " AND "
        ]
        CAST_AND_TYPE_MAP = [
            {
                "type": int,
                "cast": lambda v: "::numeric",
                "value": lambda v: unicode(v),
                "type_check" : lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": float,
                "cast": lambda v: "::numeric",
                "value": lambda v: unicode(v),
                "type_check": lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": basestring,
                "cast": lambda v: "",
                "value": lambda v: u"'{}'".format(escape_single_quote(v)),
                "type_check": lambda v: ["", ""],
            },
            {
                "type": list,
                "cast": lambda v: get_cast_attr(v[0], "cast"),
                "value": lambda v: format_list(v),
                "type_check": lambda v: get_cast_attr(v[0], "type_check"),
            },
        ]

        return get_cast_attr(val)

    def call(self, predicate, queryset):
        return queryset.extra(where=[predicate]) if predicate else queryset<commit_msg>fix: should be compared against bool value, not string<commit_after>from UserString import UserString

RE_CASE = "CASE WHEN ({0}->'{1}' ~ E'"
RE_THEN = "') THEN "
RE_ELSE = "ELSE -2147483648 END"

class ExpressionStr(UserString):
    """
    Subclass of str() that allows annotations based on which key and
    test are being used.
    """
    daff_key = ''
    daff_test = ''
    daff_val = ''

def breaks_optimizer(expr):
    if not isinstance(expr, ExpressionStr):
        return True

    return (
        expr.daff_test in {"!=", "!in"} or
        (expr.daff_test == "?=" and expr.daff_val == False) or
        (expr.daff_test == "=" and isinstance(expr.daff_val, basestring))
    )

def escape_string_sql(s):
    return "'{}'".format(s)

def make_sql_array(*strings):
    return "ARRAY[{}]".format(
        ",".join(escape_string_sql(s) for s in strings)
    )

class HStoreQueryDelegate(object):

    def __init__(self, hstore_field_name):
        self.field = hstore_field_name

    def mk_any(self, children):
        if children == []:
            return "false"

        sql_expr = " OR ".join("({})".format(child_exp) for child_exp in children)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?| {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " OR ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND ({1})".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_all(self, children):
        if children == ['']:
            return "true"

        sql_expr = " AND ".join("({})".format(child_exp) for child_exp in children if child_exp)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?& {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " AND ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND {1}".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_not_any(self, children):
        return " NOT ({0})".format(self.mk_any(children))

    def mk_not_all(self, children):
        return " NOT ({0})".format(self.mk_all(children))

    def mk_test(self, daff_test_str):
        test_str = daff_test_str

        if test_str == "?=":
            test_fn = lambda k, v, t: "{0}?{1}".format(k, v)
            test_fn.is_datapoint_test = True
        else:
            test_fn = lambda k, v, t: "{0} {1} {2}".format(k, test_str, v)
            if test_str == "!=":
                test_fn.is_NE_test = True
            elif test_str == "=":
                test_fn = lambda k, v, t: "({0} {1}))".format(k, v) if t == basestring else "{0} {1} {2}".format(k, test_str, v)
                test_fn.is_EQ_test = True
            elif test_str == "!in":
                test_fn.is_NOT_IN_test = True
                test_str = "NOT IN"
            elif test_str == "in":
                test_fn.is_IN_test = True

        test_fn.test_str = daff_test_str
        return test_fn

    def mk_cmp(self, key, val, test):
        daff_key = key
        daff_test = test.test_str
        daff_val = val

        if getattr(test, "is_datapoint_test", False):
            # here we cover:
            # [NOT] hstore_col ? '1ukmoviestudios - Disney'
            negate = "NOT " if val == False else ""
            val = "'{0}'".format(key)
            key = "{0}{1}".format(negate, self.field)
            _type = None
        else:
            is_eq_test = getattr(test, "is_EQ_test", False)
            _type, cast, val, type_check = self.cond_cast(val)

            if getattr(test, "is_NE_test", False) or getattr(test, "is_NOT_IN_test", False):
                # here we cover:
                # NOT (hstore_col?'wrong attribute') OR (hstore_col->'wrong attribute')::integer != 2
                key_format = "NOT ({0}?'{1}') OR %s {3} ({0}->'{1}'){2}"
                # if its cast - exclude those not matching type
                key_format = key_format % (" NOT " + type_check[0] + " OR " if cast else "")

            elif is_eq_test and _type == basestring:
                # here we convert '=' to '@>'
                # instead of:
                #   (hs_data->'univisionlanguage1') = 'both'
                # we do:
                #   hs_data @> hstore('univisionlanguage1', 'both')
                key_format = "{0} @> hstore('{1}',"

            elif getattr(test, "is_IN_test", False) or is_eq_test:
                # here we convert '=' to '? AND =':
                # hs_answers?'industries - luxury' AND hs_answers->'industries - luxury' = 'yes'
                key_format = "({0}?'{1}') AND {3} ({0}->'{1}'){2}"

            else:
                key_format = "{3} ({0}->'{1}'){2} "

            key = key_format.format(self.field, key, cast, "".join(type_check)).format(self.field, key)

        expr = ExpressionStr(test(key, val, _type))
        expr.daff_key = daff_key
        expr.daff_test = daff_test
        expr.daff_val = daff_val

        return expr

    def cond_cast(self, val):
        def escape_single_quote(val):
            if isinstance(val, basestring):
                return val.replace(u"'", u"''")
            return val

        def format_list(lst):
            delimiter = "'" if isinstance(lst[0], basestring) else ""
            formatted_list = ",".join(
                [u"{1}{0}{1}".format(escape_single_quote(elem), delimiter) for elem in lst]
            )
            return u"({0})".format(formatted_list)

        def get_cast_attr(val, attr=None):
            for m in CAST_AND_TYPE_MAP:
                if isinstance(val, m["type"]):

                    if attr:
                        return m[attr](val)

                    return [m["type"]] + [m[a](val) for a in ["cast", "value", "type_check"]]

        NUMERIC_TYPE_CHECK = [
                "({0}->'{1}') ~ E'^(?=.+)(?:[1-9]\\\d*|0)?(?:\\\.\\\d+)?$'",
                " AND "
        ]
        CAST_AND_TYPE_MAP = [
            {
                "type": int,
                "cast": lambda v: "::numeric",
                "value": lambda v: unicode(v),
                "type_check" : lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": float,
                "cast": lambda v: "::numeric",
                "value": lambda v: unicode(v),
                "type_check": lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": basestring,
                "cast": lambda v: "",
                "value": lambda v: u"'{}'".format(escape_single_quote(v)),
                "type_check": lambda v: ["", ""],
            },
            {
                "type": list,
                "cast": lambda v: get_cast_attr(v[0], "cast"),
                "value": lambda v: format_list(v),
                "type_check": lambda v: get_cast_attr(v[0], "type_check"),
            },
        ]

        return get_cast_attr(val)

    def call(self, predicate, queryset):
        return queryset.extra(where=[predicate]) if predicate else queryset<|endoftext|>


<commit_before>from UserString import UserString

RE_CASE = "CASE WHEN ({0}->'{1}' ~ E'"
RE_THEN = "') THEN "
RE_ELSE = "ELSE -2147483648 END"

class ExpressionStr(UserString):
    """
    Subclass of str() that allows annotations based on which key and
    test are being used.
    """
    daff_key = ''
    daff_test = ''
    daff_val = ''

def breaks_optimizer(expr):
    if not isinstance(expr, ExpressionStr):
        return True

    return (
        expr.daff_test in {"!=", "!in"} or
        (expr.daff_test == "?=" and expr.daff_val == "false")
    )

def escape_string_sql(s):
    return "'{}'".format(s)

def make_sql_array(*strings):
    return "ARRAY[{}]".format(
        ",".join(escape_string_sql(s) for s in strings)
    )

class HStoreQueryDelegate(object):

    def __init__(self, hstore_field_name):
        self.field = hstore_field_name

    def mk_any(self, children):
        if children == []:
            return "false"

        sql_expr = " OR ".join("({})".format(child_exp) for child_exp in children)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?| {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " OR ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND ({1})".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_all(self, children):
        if children == ['']:
            return "true"

        sql_expr = " AND ".join("({})".format(child_exp) for child_exp in children if child_exp)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?& {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " AND ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND {1}".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_not_any(self, children):
        return " NOT ({0})".format(self.mk_any(children))

    def mk_not_all(self, children):
        return " NOT ({0})".format(self.mk_all(children))

    def mk_test(self, daff_test_str):
        test_str = daff_test_str

        if test_str == "?=":
            test_fn = lambda k, v: "{0}?{1}".format(k, v)
            test_fn.is_datapoint_test = True
        else:
            test_fn = lambda k, v: "{0} {1} {2}".format(k, test_str, v)
            if test_str == "!=":
                test_fn.is_NE_test = True
            elif test_str == "=":
                test_fn = lambda k, v: "({0} {1}))".format(k, "'{}'".format(v) if isinstance(v, (int, float) ) else v)
                test_fn.is_EQ_test = True
            elif test_str == "!in":
                test_fn.is_NOT_IN_test = True
                test_str = "NOT IN"
            elif test_str == "in":
                test_fn.is_IN_test = True

        test_fn.test_str = daff_test_str
        return test_fn

    def mk_cmp(self, key, val, test):
        daff_key = key
        daff_test = test.test_str
        daff_val = val

        if getattr(test, "is_datapoint_test", False):
            # here we cover:
            # [NOT] hstore_col ? '1ukmoviestudios - Disney'
            negate = "NOT " if val == False else ""
            val = "'{0}'".format(key)
            key = "{0}{1}".format(negate, self.field)
        else:
            is_eq_test = getattr(test, "is_EQ_test", False)
            cast, val, type_check = self.cond_cast(val, is_eq_test)

            if getattr(test, "is_NE_test", False) or getattr(test, "is_NOT_IN_test", False):
                # here we cover:
                # NOT (hstore_col?'wrong attribute') OR (hstore_col->'wrong attribute')::integer != 2
                key_format = "NOT ({0}?'{1}') OR %s {3} ({0}->'{1}'){2}"
                # if its cast - exclude those not matching type
                key_format = key_format % (" NOT " + type_check[0] + " OR " if cast else "")

            elif is_eq_test:
                # here we convert '=' to '@>'
                # instead of:
                #   (hs_data->'univisionlanguage1') = 'both'
                # we do:
                #   hs_data @> hstore('univisionlanguage1', 'both')
                key_format = "{0} @> hstore('{1}',"

            elif getattr(test, "is_IN_test", False) or is_eq_test:
                # here we convert '=' to '? AND =':
                # hs_answers?'industries - luxury' AND hs_answers->'industries - luxury' = 'yes'
                key_format = "({0}?'{1}') AND {3} ({0}->'{1}'){2}"

            else:
                key_format = "{3} ({0}->'{1}'){2} "

            key = key_format.format(self.field, key, cast, "".join(type_check)).format(self.field, key)

        expr = ExpressionStr(test(key, val))
        expr.daff_key = daff_key
        expr.daff_test = daff_test
        expr.daff_val = daff_val

        return expr

    def cond_cast(self, val, is_eq_test):
        def format_list(lst):
            delimiter = "'" if isinstance(lst[0], basestring) else ""
            formatted_list = ",".join(
                [u"{1}{0}{1}".format(elem, delimiter) for elem in lst]
            )
            return u"({0})".format(formatted_list)

        def get_cast_attr(val, attr=None):
            for m in CAST_AND_TYPE_MAP:
                if isinstance(val, m["type"]):

                    if attr:
                        return m[attr](val)

                    return tuple([m[a](val) for a in ["cast", "value", "type_check"]])

        NUMERIC_TYPE_CHECK = [
                "({0}->'{1}') ~ E'^(?=.+)(?:[1-9]\\\d*|0)?(?:\\\.\\\d+)?$'",
                " AND "
        ]

        numeric_value_conv = lambda v: u"'{0}'".format(v) if is_eq_test else unicode(v)
        CAST_AND_TYPE_MAP = [
            {
                "type": int,
                "cast": lambda v: "::numeric",
                "value": numeric_value_conv,
                "type_check" : lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": float,
                "cast": lambda v: "::numeric",
                "value": numeric_value_conv,
                "type_check": lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": basestring,
                "cast": lambda v: "",
                "value": lambda v: u"'{0}'".format(v),
                "type_check": lambda v: ["", ""],
            },
            {
                "type": list,
                "cast": lambda v: get_cast_attr(v[0], "cast"),
                "value": lambda v: format_list(v),
                "type_check": lambda v: get_cast_attr(v[0], "type_check"),
            },
        ]

        return get_cast_attr(val)

    def call(self, predicate, queryset):
        return queryset.extra(where=[predicate]) if predicate else queryset
<commit_msg>do '@> check' only for strings<commit_after>from UserString import UserString

RE_CASE = "CASE WHEN ({0}->'{1}' ~ E'"
RE_THEN = "') THEN "
RE_ELSE = "ELSE -2147483648 END"

class ExpressionStr(UserString):
    """
    Subclass of str() that allows annotations based on which key and
    test are being used.
    """
    daff_key = ''
    daff_test = ''
    daff_val = ''

def breaks_optimizer(expr):
    if not isinstance(expr, ExpressionStr):
        return True

    return (
        expr.daff_test in {"!=", "!in"} or
        (expr.daff_test == "?=" and expr.daff_val == "false")
    )

def escape_string_sql(s):
    return "'{}'".format(s)

def make_sql_array(*strings):
    return "ARRAY[{}]".format(
        ",".join(escape_string_sql(s) for s in strings)
    )

class HStoreQueryDelegate(object):

    def __init__(self, hstore_field_name):
        self.field = hstore_field_name

    def mk_any(self, children):
        if children == []:
            return "false"

        sql_expr = " OR ".join("({})".format(child_exp) for child_exp in children)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?| {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " OR ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND ({1})".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_all(self, children):
        if children == ['']:
            return "true"

        sql_expr = " AND ".join("({})".format(child_exp) for child_exp in children if child_exp)

        if any(breaks_optimizer(child_exp) for child_exp in children):
            return sql_expr

        keys = {child_exp.daff_key for child_exp in children}
        if len(keys) <= 1:
            return sql_expr

        optimization_expr = "{field} ?& {keys}".format(
            field=self.field,
            keys=make_sql_array(*keys)
        )

        sql_expr = " AND ".join("({})".format(child_exp)
                               for child_exp in children
                               if child_exp.daff_test != "?="
                               )
        if sql_expr:
            return "{0} AND {1}".format(optimization_expr, sql_expr)
        else:
            return optimization_expr

    def mk_not_any(self, children):
        return " NOT ({0})".format(self.mk_any(children))

    def mk_not_all(self, children):
        return " NOT ({0})".format(self.mk_all(children))

    def mk_test(self, daff_test_str):
        test_str = daff_test_str

        if test_str == "?=":
            test_fn = lambda k, v, t: "{0}?{1}".format(k, v)
            test_fn.is_datapoint_test = True
        else:
            test_fn = lambda k, v, t: "{0} {1} {2}".format(k, test_str, v)
            if test_str == "!=":
                test_fn.is_NE_test = True
            elif test_str == "=":
                test_fn = lambda k, v, t: "({0} {1}))".format(k, v) if t == basestring else "{0} {1} {2}".format(k, test_str, v)
                test_fn.is_EQ_test = True
            elif test_str == "!in":
                test_fn.is_NOT_IN_test = True
                test_str = "NOT IN"
            elif test_str == "in":
                test_fn.is_IN_test = True

        test_fn.test_str = daff_test_str
        return test_fn

    def mk_cmp(self, key, val, test):
        daff_key = key
        daff_test = test.test_str
        daff_val = val

        if getattr(test, "is_datapoint_test", False):
            # here we cover:
            # [NOT] hstore_col ? '1ukmoviestudios - Disney'
            negate = "NOT " if val == False else ""
            val = "'{0}'".format(key)
            key = "{0}{1}".format(negate, self.field)
            _type = None
        else:
            is_eq_test = getattr(test, "is_EQ_test", False)
            _type, cast, val, type_check = self.cond_cast(val)

            if getattr(test, "is_NE_test", False) or getattr(test, "is_NOT_IN_test", False):
                # here we cover:
                # NOT (hstore_col?'wrong attribute') OR (hstore_col->'wrong attribute')::integer != 2
                key_format = "NOT ({0}?'{1}') OR %s {3} ({0}->'{1}'){2}"
                # if its cast - exclude those not matching type
                key_format = key_format % (" NOT " + type_check[0] + " OR " if cast else "")

            elif is_eq_test and _type == basestring:
                # here we convert '=' to '@>'
                # instead of:
                #   (hs_data->'univisionlanguage1') = 'both'
                # we do:
                #   hs_data @> hstore('univisionlanguage1', 'both')
                key_format = "{0} @> hstore('{1}',"

            elif getattr(test, "is_IN_test", False) or is_eq_test:
                # here we convert '=' to '? AND =':
                # hs_answers?'industries - luxury' AND hs_answers->'industries - luxury' = 'yes'
                key_format = "({0}?'{1}') AND {3} ({0}->'{1}'){2}"

            else:
                key_format = "{3} ({0}->'{1}'){2} "

            key = key_format.format(self.field, key, cast, "".join(type_check)).format(self.field, key)

        expr = ExpressionStr(test(key, val, _type))
        expr.daff_key = daff_key
        expr.daff_test = daff_test
        expr.daff_val = daff_val

        return expr

    def cond_cast(self, val):
        def format_list(lst):
            delimiter = "'" if isinstance(lst[0], basestring) else ""
            formatted_list = ",".join(
                [u"{1}{0}{1}".format(elem, delimiter) for elem in lst]
            )
            return u"({0})".format(formatted_list)

        def get_cast_attr(val, attr=None):
            for m in CAST_AND_TYPE_MAP:
                if isinstance(val, m["type"]):

                    if attr:
                        return m[attr](val)

                    return [m["type"]] + [m[a](val) for a in ["cast", "value", "type_check"]]

        NUMERIC_TYPE_CHECK = [
                "({0}->'{1}') ~ E'^(?=.+)(?:[1-9]\\\d*|0)?(?:\\\.\\\d+)?$'",
                " AND "
        ]
        CAST_AND_TYPE_MAP = [
            {
                "type": int,
                "cast": lambda v: "::numeric",
                "value": lambda v: unicode(v),
                "type_check" : lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": float,
                "cast": lambda v: "::numeric",
                "value": lambda v: unicode(v),
                "type_check": lambda v: NUMERIC_TYPE_CHECK,
            },
            {
                "type": basestring,
                "cast": lambda v: "",
                "value": lambda v: u"'{0}'".format(v),
                "type_check": lambda v: ["", ""],
            },
            {
                "type": list,
                "cast": lambda v: get_cast_attr(v[0], "cast"),
                "value": lambda v: format_list(v),
                "type_check": lambda v: get_cast_attr(v[0], "type_check"),
            },
        ]

        return get_cast_attr(val)

    def call(self, predicate, queryset):
        return queryset.extra(where=[predicate]) if predicate else queryset
<|endoftext|>


<commit_before># numPy uses C to compute things and then passes to python
# this makes numPy very fast

import numpy as np

def test_run():
    # convert list to 1D array
    np.array([2, 3, 4])
    np.array([(2,3,4), (5,6,7)])

# empty arrays actually initialize with random values dependent on current memory state
    # 1D array
    np.empty(5)
    # 2D array
    np.empty((5,4))
    # 3D array
    np.empty((5,4,3))
    # 2D array with all values equal to 1
    np.ones((5,4), dtype='i')
    # default values are float, using dtype to change
    np.array([1, 2, 3], dtype='i')
    # generate random number array sampled from [0.0, 1.0]
    np.random.random((5,4)) # pass in a tuple
    np.random.rand(5,4) # not a tuple
    # sample numbers from a gaussian distribution with mean=0 SD1=1
    x = np.random.normal(loc=0, scale=1, size=(2,3))
    np.random.normal
    np.mean(x)
    np.std(x)
    # generate random integers
    np.random.randint(10) # single integer between 0-10
    np.random.randint(0, 10) # same as above
    np.random.randint(0, 10, size=5) # integers between 0-10 filling 1D array
    np.random.randint(0, 10, size=(2, 4)) # 2x3 array of 0-10 integers

# reshaping arrays
    a = np.random.random((5, 4)) # 5x4 array of random numbers
    a
    a.shape # displays array dimensions
    a.shape[0] # displays number of rows
    a.shape[1] # displays number of columns
    len(a.shape) # displays number of dimensions
    a.size # number of elements in array
    a.dtype # shows type

    # operations on ndarrays
    np.random.seed(693) # seed the random number generator
    np.random.randint(1000000) # now always gives the same number
    a = np.random.randint(0, 10, size=(5, 4)) # 5x4 random int in 0-10
    #print "Array:\n", a
    #print "Sum of all elements:", a.sum() # sums all array elements

    # iterate over rows to compute sum of each column
    a.sum(axis=0)
    # iterate over rows to compute sum of each row
    a.sum(axis=1)
    # min/max/mean
    a.min(axis=0) # min of each column
    a.max(axis=1) # max of each row
    a.mean() # mean of all elements

    # searching for the index of elements
    # http://docs.scipy.org/doc/numpy/reference/routines.sort.html
    a
    np.argmax(a) # only prints first instance of max val

# accessing arrary elements
    a
    element = a[3, 2] # [row, column]
    element

# using slicing
    a[0, 1:3] # row zero columns 1-2
    a[0:2, 0:2] # rows 0-1 columns 0-1
    a[:, 0:3:2] # every row, columns 0-3 not including 3 in steps of 2

# assigning values to specific array locations
    a[0, 0] = 1 # change the 0, 0 value to 1
    a[0,:] = 2 # assign the entire first row a value of 2
    a[:, 3] = [1, 2, 3, 4, 5] # assign list values to the 4th column
    a

# indexing an array with another array
# basically just allows multiple positions to be accessed at once
    indices = np.array([1,1,2,3]) # multidimensional arrays can also be used
    a = np.random.rand(5)
    a, '\n'
    a[indices]

# indexing using boolean arrays
    a = np.array([(20,25,10,23,26,32,10,5,0),(0,2,50,20,0,1,28,5,0)])
    mean = a.mean()
    # for loop could be used to look for all values less than the mean
    # or masking can be used to boolean compare directly
    a[a<mean]

if __name__ == "__main__":
    test_run()
<commit_msg>update<commit_after># numPy uses C to compute things and then passes to python
# this makes numPy very fast

import numpy as np

def test_run():
    # convert list to 1D array
    np.array([2, 3, 4])
    np.array([(2,3,4), (5,6,7)])

# empty arrays actually initialize with random values dependent on current memory state
    # 1D array
    np.empty(5)
    # 2D array
    np.empty((5,4))
    # 3D array
    np.empty((5,4,3))
    # 2D array with all values equal to 1
    np.ones((5,4), dtype='i')
    # default values are float, using dtype to change
    np.array([1, 2, 3], dtype='i')
    # generate random number array sampled from [0.0, 1.0]
    np.random.random((5,4)) # pass in a tuple
    np.random.rand(5,4) # not a tuple
    # sample numbers from a gaussian distribution with mean=0 SD1=1
    x = np.random.normal(loc=0, scale=1, size=(2,3))
    np.random.normal
    np.mean(x)
    np.std(x)
    # generate random integers
    np.random.randint(10) # single integer between 0-10
    np.random.randint(0, 10) # same as above
    np.random.randint(0, 10, size=5) # integers between 0-10 filling 1D array
    np.random.randint(0, 10, size=(2, 4)) # 2x3 array of 0-10 integers

# reshaping arrays
    a = np.random.random((5, 4)) # 5x4 array of random numbers
    a
    a.shape # displays array dimensions
    a.shape[0] # displays number of rows
    a.shape[1] # displays number of columns
    len(a.shape) # displays number of dimensions
    a.size # number of elements in array
    a.dtype # shows type

    # operations on ndarrays
    np.random.seed(693) # seed the random number generator
    np.random.randint(1000000) # now always gives the same number
    a = np.random.randint(0, 10, size=(5, 4)) # 5x4 random int in 0-10
    #print "Array:\n", a
    #print "Sum of all elements:", a.sum() # sums all array elements

    # iterate over rows to compute sum of each column
    a.sum(axis=0)
    # iterate over rows to compute sum of each row
    a.sum(axis=1)
    # min/max/mean
    a.min(axis=0) # min of each column
    a.max(axis=1) # max of each row
    a.mean() # mean of all elements

    # searching for the index of elements
    # http://docs.scipy.org/doc/numpy/reference/routines.sort.html
    a
    np.argmax(a) # only prints first instance of max val

# accessing arrary elements
    a
    element = a[3, 2] # [row, column]
    element

# using slicing
    a[0, 1:3] # row zero columns 1-2
    a[0:2, 0:2] # rows 0-1 columns 0-1
    a[:, 0:3:2] # every row, columns 0-3 not including 3 in steps of 2

# assigning values to specific array locations
    a[0, 0] = 1 # change the 0, 0 value to 1
    a[0,:] = 2 # assign the entire first row a value of 2
    a[:, 3] = [1, 2, 3, 4, 5] # assign list values to the 4th column
    a

# indexing an array with another array
# basically just allows multiple positions to be accessed at once
    indices = np.array([1,1,2,3]) # multidimensional arrays can also be used
    a = np.random.rand(5)
    a, '\n'
    a[indices]

# indexing using boolean arrays
    a = np.array([(20,25,10,23,26,32,10,5,0),(0,2,50,20,0,1,28,5,0)])
    mean = a.mean()
    # for loop could be used to look for all values less than the mean
    # or masking can be used to boolean compare directly
    a[a<mean]

# arithmetic operations on arrays
    a = np.array([(1,2,3,4,5), (10,20,30,40,50)])
    b = np.array([(100,200,300,400,500), (1,2,3,4,5)], dtype='f')
    a
    a * 2
    a / 2.0
    a + b
    a * b # multiplies each element together
    print a / b

if __name__ == "__main__":
    test_run()
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
import cv2
import numpy as np

import digit_detector.region_proposal as rp
import digit_detector.show as show

def load_model(filename):
    from keras.models import load_model
    model = load_model(filename)
    return model

img_files = ['imgs/1.png', 'imgs/2.png']

if __name__ == "__main__":
    # 1. image files
    img_file = img_files[1]

    # 2. image
    img = cv2.imread(img_file)
    #img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # 3. region proposals (N, 32, 32, 3)
    patches = rp.propose_patches(img)
    temp = patches

    # 4. Convert to gray
    patches = [cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY) for patch in patches]
    patches = np.array(patches)
    patches = patches.reshape(-1, 32, 32, 1)
    print patches.shape

    # 5. Run pre-trained classifier
    model = load_model("detector_model.hdf5")

    probs = model.predict_proba(patches)[:, 1]

    print probs.shape

    show.plot_images(temp, probs.tolist())

<commit_msg>original image 에서 bounding-box 별 확률을 plot<commit_after>#-*- coding: utf-8 -*-
import cv2
import numpy as np

import digit_detector.region_proposal as rp
import digit_detector.show as show

def load_model(filename):
    from keras.models import load_model
    model = load_model(filename)
    return model

img_files = ['imgs/1.png', 'imgs/2.png']

if __name__ == "__main__":
    # 1. image files
    img_file = img_files[1]

    # 2. image
    img = cv2.imread(img_file)
    #img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # 3. region proposals (N, 32, 32, 3)
    patches, bbs = rp.propose_patches(img)
    temp = patches

    # 4. Convert to gray
    patches = [cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY) for patch in patches]
    patches = np.array(patches)
    patches = patches.reshape(-1, 32, 32, 1)
    print patches.shape

    # 5. Run pre-trained classifier
    model = load_model("detector_model.hdf5")

    probs = model.predict_proba(patches)[:, 1]

    print probs.shape

    show.plot_images(temp, probs.tolist())
    show.plot_bounding_boxes(img, bbs, probs.tolist())

<|endoftext|>