<commit_before>__all__ = ['record', 'recarray','format_parser']

import numeric as sb
import numerictypes as nt
import sys
import types
import re, stat, os
import _internal

_byteorderconv = {'b':'>',
                  'l':'<',
                  'n':'=',
                  'B':'>',
                  'L':'<',
                  'N':'=',
                  'S':'s',
                  's':'s',
                  '>':'>',
                  '<':'<',
                  '=':'=',
                  '|':'|',
                  'I':'|',
                  'i':'|'}

_sysbyte = _byteorderconv[sys.byteorder[0]]
_byteorders = ['>','<','=']

# formats regular expression
# allows multidimension spec with a tuple syntax in front
# of the letter code '(2,3)f4' and ' (  2 ,  3  )  f4  '
# are equally allowed

numfmt = nt.typeDict
_typestr = nt._typestr

def find_duplicate(list):
    """Find duplication in a list, return a list of duplicated elements"""
    dup = []
    for i in range(len(list)):
        if (list[i] in list[i+1:]):
            if (list[i] not in dup):
                dup.append(list[i])
    return dup

class format_parser:
    def __init__(self, formats, names, titles, aligned=False):
        self._parseFormats(formats, aligned)
        self._setfieldnames(names, titles)
        self._createdescr()

    def _parseFormats(self, formats, aligned=0):
        """ Parse the field formats """

        dtypedescr = sb.dtypedescr(formats, aligned)
        fields = dtypedescr.fields
        keys = fields[-1]
        self._f_formats = [fields[key][0] for key in keys]
        self._offsets = [fields[key][1] for key in keys]
        self._nfields = len(keys)

    def _setfieldnames(self, names, titles):
        """convert input field names into a list and assign to the _names
        attribute """

        if (names):
            if (type(names) in [types.ListType, types.TupleType]):
                pass
            elif (type(names) == types.StringType):
                names = names.split(',')
            else:
                raise NameError, "illegal input names %s" % `names`

            self._names = map(lambda n:n.strip(), names)[:self._nfields]
        else:
            self._names = []

        # if the names are not specified, they will be assigned as "f1, f2,..."
        # if not enough names are specified, they will be assigned as "f[n+1],
        # f[n+2],..." etc. where n is the number of specified names..."
        self._names += map(lambda i:
            'f'+`i`, range(len(self._names)+1,self._nfields+1))

        # check for redundant names
        _dup = find_duplicate(self._names)
        if _dup:
            raise ValueError, "Duplicate field names: %s" % _dup

        if (titles):
            self._titles = [n.strip() for n in titles][:self._nfields]
        else:
            self._titles = []
            titles = []

        if (self._nfields > len(titles)):
            self._titles += [None]*(self._nfields-len(titles))

    def _createdescr(self):
        self._descr = sb.dtypedescr({'names':self._names,
                                     'formats':self._f_formats,
                                     'offsets':self._offsets,
                                     'titles':self._titles})

class record(nt.void):
    def __repr__(self):
        return self.__str__()

    def __str__(self):
        return str(self.item())

    def __getattribute__(self, attr):
        if attr in ['setfield', 'getfield', 'fields']:
            return nt.void.__getattribute__(self, attr)
        fielddict = nt.void.__getattribute__(self, 'fields')
        res = fielddict.get(attr,None)
        if res:
            return self.getfield(*res[:2])
        return nt.void.__getattribute__(self, attr)

    def __setattr__(self, attr, val):
        if attr in ['setfield', 'getfield', 'fields']:
            raise AttributeError, "Cannot set '%s' attribute" % attr;
        fielddict = nt.void.__getattribute__(self,'fields')
        res = fielddict.get(attr,None)
        if res:
            return self.setfield(val,*res[:2])

        return nt.void.__setattr__(self,attr,val)

    def __getitem__(self, obj):
        return self.getfield(*(self.fields[obj][:2]))

    def __setitem__(self, obj, val):
        return self.setfield(val, *(self.fields[obj][:2]))

# The recarray is almost identical to a standard array (which supports
#   named fields already)  The biggest difference is that it can use
#   attribute-lookup to find the fields and it returns a record item.

# If byteorder is given it forces a particular byteorder on all
#  the fields (and any subfields)

class recarray(sb.ndarray):
    def __new__(subtype, shape, formats, names=None, titles=None,
                buf=None, offset=0, strides=None, byteorder=None,
                aligned=0):

        if isinstance(formats, sb.dtypedescr):
            descr = formats
        else:
            parsed = format_parser(formats, names, titles, aligned)
            descr = parsed._descr

        if (byteorder is not None):
            byteorder = _byteorderconv[byteorder[0]]
            descr = descr.newbyteorder(byteorder)

        if buf is None:
            self = sb.ndarray.__new__(subtype, shape, (record, descr))
        else:
            self = sb.ndarray.__new__(subtype, shape, (record, descr),
                                      buffer=buf, offset=offset)
        return self

    def __getattribute__(self, attr):
        fielddict = sb.ndarray.__getattribute__(self,'dtypedescr').fields
        try:
            res = fielddict[attr][:2]
        except:
            return sb.ndarray.__getattribute__(self,attr)

        return self.getfield(*res)

    def __setattr__(self, attr, val):
        fielddict = sb.ndarray.__getattribute__(self,'dtypedescr').fields
        try:
            res = fielddict[attr][:2]
        except:
            return sb.ndarray.__setattr__(self,attr,val)

        return self.setfield(val,*res)

def fromarrays(arrayList, formats=None, names=None, titles=None, shape=None,
               aligned=0):
    """ create a record array from a (flat) list of arrays

    >>> x1=array([1,2,3,4])
    >>> x2=array(['a','dd','xyz','12'])
    >>> x3=array([1.1,2,3,4])
    >>> r=fromarrays([x1,x2,x3],names='a,b,c')
    >>> print r[1]
    (2, 'dd\x00', 2.0)
    >>> x1[1]=34
    >>> r.a
    array([1, 2, 3, 4])
    """

    if shape is None or shape == 0:
        shape = arrayList[0].shape

    if isinstance(shape, int):
        shape = (shape,)

    if formats is None:
        # go through each object in the list to see if it is an ndarray
        # and determine the formats.
        formats = ''
        for obj in arrayList:
            if not isinstance(obj, sb.ndarray):
                raise ValueError, "item in the array list must be an ndarray."
            if obj.ndim == 1:
                _repeat = ''
            elif len(obj._shape) >= 2:
                _repeat = `obj._shape[1:]`
            formats += _repeat + _typestr[obj.dtype]
            if issubclass(obj.dtype, nt.flexible):
                formats += `obj.itemsize`
            formats += ','
        formats=formats[:-1]

    for obj in arrayList:
        if obj.shape != shape:
            raise ValueError, "array has different shape"

    parsed = format_parser(formats, names, titles, aligned)
    _names = parsed._names
    _array = recarray(shape, parsed._descr)

    # populate the record array (makes a copy)
    for i in range(len(arrayList)):
        _array[_names[i]] = arrayList[i]

    return _array

# shape must be 1-d
def fromrecords(recList, formats=None, names=None, titles=None, shape=None,
                aligned=0):
    """ create a Record Array from a list of records in text form

        The data in the same field can be heterogeneous, they will be promoted
        to the highest data type.  This method is intended for creating
        smaller record arrays.  If used to create large array e.g.

        r=fromrecords([[2,3.,'abc']]*100000)

        it is slow.

    >>> r=fromrecords([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
    >>> print r[0]
    (456, 'dbe', 1.2)
    >>> r.col1
    array([456,   2])
    >>> r.col2
    array(['dbe', 'de'])
    >>> import cPickle
    >>> print cPickle.loads(cPickle.dumps(r))
    recarray[
    (456, 'dbe', 1.2),
    (2, 'de', 1.3)
    ]
    """

    if (shape is None or shape == 0):
        shape = len(recList)

    if isinstance(shape, (int, long)):
        shape = (shape,)

    if len(shape) > 1:
        raise ValueError, "Can only deal with 1-d list of records"

    nfields = len(recList[0])
    if formats is None:  # slower
        obj = sb.array(recList,dtype=object)
        arrlist = [sb.array(obj[:,i].tolist()) for i in xrange(nfields)]
        return fromarrays(arrlist, formats=formats, shape=shape, names=names,
                          titles=titles, aligned=aligned)

    parsed = format_parser(formats, names, titles, aligned)
    _names = parsed._names
    _array = recarray(shape, parsed._descr)

    for k in xrange(_array.size):
        _array[k] = tuple(recList[k])

    return _array

def fromstring(datastring, formats, shape=None, names=None, titles=None,
               byteorder=None, aligned=0, offset=0):
    """ create a (read-only) record array from binary data contained in
    a string"""

    parsed = format_parser(formats, names, titles, aligned)
    itemsize = parsed._descr.itemsize
    if (shape is None or shape == 0 or shape == -1):
        shape = (len(datastring)-offset) / itemsize

    _array = recarray(shape, parsed._descr, names=names,
                      titles=titles, buf=datastring, offset=offset,
                      byteorder=byteorder)
    return _array

def fromfile(fd, formats, shape=None, names=None, titles=None,
             byteorder=None, aligned=0, offset=0):
    """Create an array from binary file data

    If file is a string then that file is opened, else it is assumed
    to be a file object. No options at the moment, all file positioning
    must be done prior to this function call with a file object

    >>> import testdata, sys
    >>> fd=open(testdata.filename)
    >>> fd.seek(2880*2)
    >>> r=fromfile(fd, formats='f8,i4,a5', shape=3, byteorder='big')
    >>> print r[0]
    (5.1000000000000005, 61, 'abcde')
    >>> r._shape
    (3,)
    """

    if (shape is None or shape == 0):
        shape = (-1,)
    elif isinstance(shape, (int, long)):
        shape = (shape,)

    name = 0
    if isinstance(fd, str):
        name = 1
        fd = open(fd, 'rb')
    if (offset > 0):
        fd.seek(offset, 1)
    try:
        size = os.fstat(fd.fileno())[stat.ST_SIZE] - fd.tell()
    except:
        size = os.path.getsize(fd.name) - fd.tell()

    parsed = format_parser(formats, names, titles, aligned)
    itemsize = parsed._descr.itemsize

    shapeprod = sb.array(shape).prod()
    shapesize = shapeprod*itemsize
    if shapesize < 0:
        shape = list(shape)
        shape[ shape.index(-1) ] = size / -shapesize
        shape = tuple(shape)
        shapeprod = sb.array(shape).prod()

    nbytes = shapeprod*itemsize

    if nbytes > size:
        raise ValueError(
                "Not enough bytes left in file for specified shape and type")

    # create the array
    _array = recarray(shape, parsed._descr)
    nbytesread = fd.readinto(_array.data)
    if nbytesread != nbytes:
        raise IOError("Didn't read as many bytes as expected")
    if name:
        fd.close()

    return _array

def array(obj, formats=None, names=None, titles=None, shape=None,
          byteorder=None, aligned=0, offset=0, strides=None):

    if isinstance(obj, (type(None), str, file)) and (formats is None):
        raise ValueError("Must define formats if object is "\
                         "None, string, or a file pointer")

    elif obj is None:
        if shape is None:
            raise ValueError("Must define a shape if obj is None")
        return recarray(shape, formats, names=names, titles=titles,
                        buf=obj, offset=offset, strides=strides,
                        byteorder=byteorder, aligned=aligned)
    elif isinstance(obj, str):
        return fromstring(obj, formats, names=names, titles=titles,
                          shape=shape, byteorder=byteorder, aligned=aligned,
                          offset=offset)
    elif isinstance(obj, (list, tuple)):
        if isinstance(obj[0], sb.ndarray):
            return fromarrays(obj, formats=formats, names=names, titles=titles,
                              shape=shape, aligned=aligned)
        else:
            return fromrecords(obj, formats=formats, names=names, titles=titles,
                               shape=shape, aligned=aligned)
    elif isinstance(obj, recarray):
        new = obj.copy()
        parsed = format_parser(formats, names, titles, aligned)
        new.dtypedescr = parsed._descr
        return new
    elif isinstance(obj, file):
        return fromfile(obj, formats=formats, names=names, titles=titles,
                        shape=shape, byteorder=byteorder, aligned=aligned,
                        offset=offset)
    else:
        raise ValueError("Unknown input type")

<commit_msg>Fixed use of byteorder.<commit_after>__all__ = ['record', 'recarray','format_parser']

import numeric as sb
import numerictypes as nt
import sys
import types
import re, stat, os
import _internal

_byteorderconv = {'b':'>',
                  'l':'<',
                  'n':'=',
                  'B':'>',
                  'L':'<',
                  'N':'=',
                  'S':'s',
                  's':'s',
                  '>':'>',
                  '<':'<',
                  '=':'=',
                  '|':'|',
                  'I':'|',
                  'i':'|'}

_sysbyte = _byteorderconv[sys.byteorder[0]]
_byteorders = ['>','<','=']

# formats regular expression
# allows multidimension spec with a tuple syntax in front
# of the letter code '(2,3)f4' and ' (  2 ,  3  )  f4  '
# are equally allowed

numfmt = nt.typeDict
_typestr = nt._typestr

def find_duplicate(list):
    """Find duplication in a list, return a list of duplicated elements"""
    dup = []
    for i in range(len(list)):
        if (list[i] in list[i+1:]):
            if (list[i] not in dup):
                dup.append(list[i])
    return dup

class format_parser:
    def __init__(self, formats, names, titles, aligned=False):
        self._parseFormats(formats, aligned)
        self._setfieldnames(names, titles)
        self._createdescr()

    def _parseFormats(self, formats, aligned=0):
        """ Parse the field formats """

        dtypedescr = sb.dtypedescr(formats, aligned)
        fields = dtypedescr.fields
        keys = fields[-1]
        self._f_formats = [fields[key][0] for key in keys]
        self._offsets = [fields[key][1] for key in keys]
        self._nfields = len(keys)

    def _setfieldnames(self, names, titles):
        """convert input field names into a list and assign to the _names
        attribute """

        if (names):
            if (type(names) in [types.ListType, types.TupleType]):
                pass
            elif (type(names) == types.StringType):
                names = names.split(',')
            else:
                raise NameError, "illegal input names %s" % `names`

            self._names = map(lambda n:n.strip(), names)[:self._nfields]
        else:
            self._names = []

        # if the names are not specified, they will be assigned as "f1, f2,..."
        # if not enough names are specified, they will be assigned as "f[n+1],
        # f[n+2],..." etc. where n is the number of specified names..."
        self._names += map(lambda i:
            'f'+`i`, range(len(self._names)+1,self._nfields+1))

        # check for redundant names
        _dup = find_duplicate(self._names)
        if _dup:
            raise ValueError, "Duplicate field names: %s" % _dup

        if (titles):
            self._titles = [n.strip() for n in titles][:self._nfields]
        else:
            self._titles = []
            titles = []

        if (self._nfields > len(titles)):
            self._titles += [None]*(self._nfields-len(titles))

    def _createdescr(self):
        self._descr = sb.dtypedescr({'names':self._names,
                                     'formats':self._f_formats,
                                     'offsets':self._offsets,
                                     'titles':self._titles})

class record(nt.void):
    def __repr__(self):
        return self.__str__()

    def __str__(self):
        return str(self.item())

    def __getattribute__(self, attr):
        if attr in ['setfield', 'getfield', 'fields']:
            return nt.void.__getattribute__(self, attr)
        fielddict = nt.void.__getattribute__(self, 'fields')
        res = fielddict.get(attr,None)
        if res:
            return self.getfield(*res[:2])
        return nt.void.__getattribute__(self, attr)

    def __setattr__(self, attr, val):
        if attr in ['setfield', 'getfield', 'fields']:
            raise AttributeError, "Cannot set '%s' attribute" % attr;
        fielddict = nt.void.__getattribute__(self,'fields')
        res = fielddict.get(attr,None)
        if res:
            return self.setfield(val,*res[:2])

        return nt.void.__setattr__(self,attr,val)

    def __getitem__(self, obj):
        return self.getfield(*(self.fields[obj][:2]))

    def __setitem__(self, obj, val):
        return self.setfield(val, *(self.fields[obj][:2]))

# The recarray is almost identical to a standard array (which supports
#   named fields already)  The biggest difference is that it can use
#   attribute-lookup to find the fields and it returns a record item.

# If byteorder is given it forces a particular byteorder on all
#  the fields (and any subfields)

class recarray(sb.ndarray):
    def __new__(subtype, shape, formats, names=None, titles=None,
                buf=None, offset=0, strides=None, byteorder=None,
                aligned=0):

        if isinstance(formats, sb.dtypedescr):
            descr = formats
        else:
            parsed = format_parser(formats, names, titles, aligned)
            descr = parsed._descr

        if (byteorder is not None):
            byteorder = _byteorderconv[byteorder[0]]
            descr = descr.newbyteorder(byteorder)

        if buf is None:
            self = sb.ndarray.__new__(subtype, shape, (record, descr))
        else:
            self = sb.ndarray.__new__(subtype, shape, (record, descr),
                                      buffer=buf, offset=offset)
        return self

    def __getattribute__(self, attr):
        fielddict = sb.ndarray.__getattribute__(self,'dtypedescr').fields
        try:
            res = fielddict[attr][:2]
        except:
            return sb.ndarray.__getattribute__(self,attr)

        return self.getfield(*res)

    def __setattr__(self, attr, val):
        fielddict = sb.ndarray.__getattribute__(self,'dtypedescr').fields
        try:
            res = fielddict[attr][:2]
        except:
            return sb.ndarray.__setattr__(self,attr,val)

        return self.setfield(val,*res)

def fromarrays(arrayList, formats=None, names=None, titles=None, shape=None,
               aligned=0):
    """ create a record array from a (flat) list of arrays

    >>> x1=array([1,2,3,4])
    >>> x2=array(['a','dd','xyz','12'])
    >>> x3=array([1.1,2,3,4])
    >>> r=fromarrays([x1,x2,x3],names='a,b,c')
    >>> print r[1]
    (2, 'dd\x00', 2.0)
    >>> x1[1]=34
    >>> r.a
    array([1, 2, 3, 4])
    """

    if shape is None or shape == 0:
        shape = arrayList[0].shape

    if isinstance(shape, int):
        shape = (shape,)

    if formats is None:
        # go through each object in the list to see if it is an ndarray
        # and determine the formats.
        formats = ''
        for obj in arrayList:
            if not isinstance(obj, sb.ndarray):
                raise ValueError, "item in the array list must be an ndarray."
            if obj.ndim == 1:
                _repeat = ''
            elif len(obj._shape) >= 2:
                _repeat = `obj._shape[1:]`
            formats += _repeat + _typestr[obj.dtype]
            if issubclass(obj.dtype, nt.flexible):
                formats += `obj.itemsize`
            formats += ','
        formats=formats[:-1]

    for obj in arrayList:
        if obj.shape != shape:
            raise ValueError, "array has different shape"

    parsed = format_parser(formats, names, titles, aligned)
    _names = parsed._names
    _array = recarray(shape, parsed._descr)

    # populate the record array (makes a copy)
    for i in range(len(arrayList)):
        _array[_names[i]] = arrayList[i]

    return _array

# shape must be 1-d
def fromrecords(recList, formats=None, names=None, titles=None, shape=None,
                aligned=0):
    """ create a Record Array from a list of records in text form

        The data in the same field can be heterogeneous, they will be promoted
        to the highest data type.  This method is intended for creating
        smaller record arrays.  If used to create large array e.g.

        r=fromrecords([[2,3.,'abc']]*100000)

        it is slow.

    >>> r=fromrecords([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
    >>> print r[0]
    (456, 'dbe', 1.2)
    >>> r.col1
    array([456,   2])
    >>> r.col2
    array(['dbe', 'de'])
    >>> import cPickle
    >>> print cPickle.loads(cPickle.dumps(r))
    recarray[
    (456, 'dbe', 1.2),
    (2, 'de', 1.3)
    ]
    """

    if (shape is None or shape == 0):
        shape = len(recList)

    if isinstance(shape, (int, long)):
        shape = (shape,)

    if len(shape) > 1:
        raise ValueError, "Can only deal with 1-d list of records"

    nfields = len(recList[0])
    if formats is None:  # slower
        obj = sb.array(recList,dtype=object)
        arrlist = [sb.array(obj[:,i].tolist()) for i in xrange(nfields)]
        return fromarrays(arrlist, formats=formats, shape=shape, names=names,
                          titles=titles, aligned=aligned)

    parsed = format_parser(formats, names, titles, aligned)
    _names = parsed._names
    _array = recarray(shape, parsed._descr)

    for k in xrange(_array.size):
        _array[k] = tuple(recList[k])

    return _array

def fromstring(datastring, formats, shape=None, names=None, titles=None,
               byteorder=None, aligned=0, offset=0):
    """ create a (read-only) record array from binary data contained in
    a string"""

    parsed = format_parser(formats, names, titles, aligned)
    itemsize = parsed._descr.itemsize
    if (shape is None or shape == 0 or shape == -1):
        shape = (len(datastring)-offset) / itemsize

    _array = recarray(shape, parsed._descr, names=names,
                      titles=titles, buf=datastring, offset=offset,
                      byteorder=byteorder)
    return _array

def fromfile(fd, formats, shape=None, names=None, titles=None,
             byteorder=None, aligned=0, offset=0):
    """Create an array from binary file data

    If file is a string then that file is opened, else it is assumed
    to be a file object. No options at the moment, all file positioning
    must be done prior to this function call with a file object

    >>> import testdata, sys
    >>> fd=open(testdata.filename)
    >>> fd.seek(2880*2)
    >>> r=fromfile(fd, formats='f8,i4,a5', shape=3, byteorder='big')
    >>> print r[0]
    (5.1000000000000005, 61, 'abcde')
    >>> r._shape
    (3,)
    """

    if (shape is None or shape == 0):
        shape = (-1,)
    elif isinstance(shape, (int, long)):
        shape = (shape,)

    name = 0
    if isinstance(fd, str):
        name = 1
        fd = open(fd, 'rb')
    if (offset > 0):
        fd.seek(offset, 1)
    try:
        size = os.fstat(fd.fileno())[stat.ST_SIZE] - fd.tell()
    except:
        size = os.path.getsize(fd.name) - fd.tell()

    parsed = format_parser(formats, names, titles, aligned)
    itemsize = parsed._descr.itemsize

    shapeprod = sb.array(shape).prod()
    shapesize = shapeprod*itemsize
    if shapesize < 0:
        shape = list(shape)
        shape[ shape.index(-1) ] = size / -shapesize
        shape = tuple(shape)
        shapeprod = sb.array(shape).prod()

    nbytes = shapeprod*itemsize

    if nbytes > size:
        raise ValueError(
                "Not enough bytes left in file for specified shape and type")

    # create the array
    _array = recarray(shape, parsed._descr, byteorder=byteorder)
    nbytesread = fd.readinto(_array.data)
    if nbytesread != nbytes:
        raise IOError("Didn't read as many bytes as expected")
    if name:
        fd.close()

    return _array

def array(obj, formats=None, names=None, titles=None, shape=None,
          byteorder=None, aligned=0, offset=0, strides=None):

    if isinstance(obj, (type(None), str, file)) and (formats is None):
        raise ValueError("Must define formats if object is "\
                         "None, string, or a file pointer")

    elif obj is None:
        if shape is None:
            raise ValueError("Must define a shape if obj is None")
        return recarray(shape, formats, names=names, titles=titles,
                        buf=obj, offset=offset, strides=strides,
                        byteorder=byteorder, aligned=aligned)
    elif isinstance(obj, str):
        return fromstring(obj, formats, names=names, titles=titles,
                          shape=shape, byteorder=byteorder, aligned=aligned,
                          offset=offset)
    elif isinstance(obj, (list, tuple)):
        if isinstance(obj[0], sb.ndarray):
            return fromarrays(obj, formats=formats, names=names, titles=titles,
                              shape=shape, aligned=aligned)
        else:
            return fromrecords(obj, formats=formats, names=names, titles=titles,
                               shape=shape, aligned=aligned)
    elif isinstance(obj, recarray):
        new = obj.copy()
        parsed = format_parser(formats, names, titles, aligned)
        new.dtypedescr = parsed._descr
        return new
    elif isinstance(obj, file):
        return fromfile(obj, formats=formats, names=names, titles=titles,
                        shape=shape, byteorder=byteorder, aligned=aligned,
                        offset=offset)
    else:
        raise ValueError("Unknown input type")

<|endoftext|>


<commit_before># Module:	dnstools
# Date:		30th June 2006
# Author:	James Mills, prologic at shortcircuit dot net dot au

"""DNS Tools

This plugin provides various tools to work with
hostnames and ip addresses.
"""

__ver__ = "0.0.3"
__author__ = "James Mills, prologic at shortcircuit dot net dot au"

import string
import socket

from kdb.plugin import BasePlugin

class DNSTools(BasePlugin):
	"""DNS Tools Plugin

	Provides commands for working with DNS (Domain Name Servers).
	See: commands dnstools
	"""

	def cmdHOST(self, source, host):
		"""Resolve the given hostname/ip

		Syntax: HOST <hostname/ip>
		"""

		isIP = True
		for c in host.replace(".", ""):
			if c not in string.digits:
				isIP = False
				break

		if isIP:
			try:
				name, aliases, addresses = socket.gethostbyaddr(
						host)
				msg = "%s -> %s" % (host, name)
			except socket.gaierror, e:
				msg = "%s -> %s" % (host, e[1])
			except socket.herror, e:
				msg = "%s -> %s" % (host, e[1])
		else:
			try:
				address = socket.gethostbyname(host)
				msg = "%s -> %s" % (host, address)
			except socket.gaierror, e:
				msg = "%s -> %s" % (host, e[1])

		return msg
<commit_msg>plugins/dnstools: Added RESOLVE command, a synonym of HOST<commit_after># Module:	dnstools
# Date:		30th June 2006
# Author:	James Mills, prologic at shortcircuit dot net dot au

"""DNS Tools

This plugin provides various tools to work with
hostnames and ip addresses.
"""

__ver__ = "0.0.4"
__author__ = "James Mills, prologic at shortcircuit dot net dot au"

import string
import socket

from kdb.plugin import BasePlugin

class DNSTools(BasePlugin):
	"""DNS Tools Plugin

	Provides commands for working with DNS (Domain Name Servers).
	See: commands dnstools
	"""

	def cmdRESOLVE(self, source, host):
		"""Synonym, of HOST

		See: HOST
		"""

		return self.cmdHOST(source, host)

	def cmdHOST(self, source, host):
		"""Resolve the given hostname/ip

		Syntax: HOST <hostname/ip>
		"""

		isIP = True
		for c in host.replace(".", ""):
			if c not in string.digits:
				isIP = False
				break

		if isIP:
			try:
				name, aliases, addresses = socket.gethostbyaddr(
						host)
				msg = "%s -> %s" % (host, name)
			except socket.gaierror, e:
				msg = "%s -> %s" % (host, e[1])
			except socket.herror, e:
				msg = "%s -> %s" % (host, e[1])
		else:
			try:
				address = socket.gethostbyname(host)
				msg = "%s -> %s" % (host, address)
			except socket.gaierror, e:
				msg = "%s -> %s" % (host, e[1])

		return msg
<|endoftext|>


<commit_before>from __future__ import absolute_import
from git.repo import Repo
from gitdb.exc import BadObject
# from kenja.detection.extract_method import detect_extract_method
# from kenja.detection.extract_method import detect_extract_method_from_commit
from kenja.detection.pull_up_method import detect_pullup_method_from_commit
from kenja.detection.pull_up_method import detect_pull_up_method
import argparse
import csv

class RefactoringDetectionCommandParser:
    def __init__(self):
        self.parser = argparse.ArgumentParser(description='Kenja a refactoring detection tool')
        self.subparsers = self.parser.add_subparsers()

        self.add_all_command()
        self.add_commits_command()

    def parse_and_execute_command(self):
        args = self.parser.parse_args()
        args.func(args)

    def add_all_command(self):
        help_str = 'detect refactoring from all commits in the Historage'
        subparser = self.subparsers.add_parser('all', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)
        subparser.set_defaults(func=self.detect_all)

    def detect_all(self, args):
        historage = Repo(args.historage_dir)
        pull_up_method_candidates = detect_pull_up_method(historage)
        print '"old_commit","new_commit","old_org_commit","new_org_commit",' \
            + '"src_method","dst_method","similarity","isSamePrameters"'
        for info in pull_up_method_candidates:
            # info[6] = sim
            info[6] = str(info[6])
            print ','.join(['"' + s + '"' for s in info])

        # candidate_revisions = set()
        # for a_commit, b_commit, org_commit, a_package, b_package, c, m, method, sim in extract_method_information:
        #     candidate_revisions.add(b_commit)
        #     print self.format_for_umldiff('jedit', a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
        #                                   sim)

        # print 'candidates:', len(extract_method_information)
        # print 'candidate revisions:', len(candidate_revisions)

    def format_for_umldiff(self, package_prefix, a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                           sim):
        target_method_info = [package_prefix]
        if a_package:
            target_method_info.append(a_package)
        target_method_info.extend((c, m))
        target_method = '.'.join(target_method_info)
        extracted_method_info = [package_prefix]
        if b_package:
            extracted_method_info.append(b_package)
        extracted_method_info.extend((c, method))
        extracted_method = '.'.join(extracted_method_info)
        return '"%s","%s","%s","%s","%s","%s"' % (a_commit, b_commit, org_commit, target_method, extracted_method, sim)

    def add_commits_command(self):
        help_str = 'detect refactoring from commits in the csv'
        subparser = self.subparsers.add_parser('commits', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)

        help_str = 'comma separated commits list. please write a_commit hash and b_commit hash per line'
        subparser.add_argument('commits_list', help=help_str)
        subparser.set_defaults(func=self.detect_from_commits_list)

    def detect_from_commits_list(self, args):
        historage = Repo(args.historage_dir)
        extract_method_information = []
        try:
            for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
                a_commit = historage.commit(a_commit_hash)
                b_commit = historage.commit(b_commit_hash)
                extract_method_information.extend(detect_pullup_method_from_commit(a_commit, b_commit))
        except ValueError:
            print "Invalid input."
            return
        except BadObject, name:
            print "Invalid hash of the commit:", name.message

        for a_commit, b_commit, org_commit, a_package, b_package, c, m, method, sim in extract_method_information:
            print self.format_for_umldiff('jedit', a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                                          sim)

def main():
    parser = RefactoringDetectionCommandParser()
    parser.parse_and_execute_command()

if __name__ == '__main__':
    main()
<commit_msg>removed unused packages<commit_after>from __future__ import absolute_import
from git.repo import Repo
from gitdb.exc import BadObject
from kenja.detection.pull_up_method import detect_pullup_method_from_commit
from kenja.detection.pull_up_method import detect_pull_up_method
import argparse
import csv

class RefactoringDetectionCommandParser:
    def __init__(self):
        self.parser = argparse.ArgumentParser(description='Kenja a refactoring detection tool')
        self.subparsers = self.parser.add_subparsers()

        self.add_all_command()
        self.add_commits_command()

    def parse_and_execute_command(self):
        args = self.parser.parse_args()
        args.func(args)

    def add_all_command(self):
        help_str = 'detect refactoring from all commits in the Historage'
        subparser = self.subparsers.add_parser('all', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)
        subparser.set_defaults(func=self.detect_all)

    def detect_all(self, args):
        historage = Repo(args.historage_dir)
        pull_up_method_candidates = detect_pull_up_method(historage)
        print '"old_commit","new_commit","old_org_commit","new_org_commit",' \
            + '"src_method","dst_method","similarity","isSamePrameters"'
        for info in pull_up_method_candidates:
            # info[6] = sim
            info[6] = str(info[6])
            print ','.join(['"' + s + '"' for s in info])

        # candidate_revisions = set()
        # for a_commit, b_commit, org_commit, a_package, b_package, c, m, method, sim in extract_method_information:
        #     candidate_revisions.add(b_commit)
        #     print self.format_for_umldiff('jedit', a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
        #                                   sim)

        # print 'candidates:', len(extract_method_information)
        # print 'candidate revisions:', len(candidate_revisions)

    def format_for_umldiff(self, package_prefix, a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                           sim):
        target_method_info = [package_prefix]
        if a_package:
            target_method_info.append(a_package)
        target_method_info.extend((c, m))
        target_method = '.'.join(target_method_info)
        extracted_method_info = [package_prefix]
        if b_package:
            extracted_method_info.append(b_package)
        extracted_method_info.extend((c, method))
        extracted_method = '.'.join(extracted_method_info)
        return '"%s","%s","%s","%s","%s","%s"' % (a_commit, b_commit, org_commit, target_method, extracted_method, sim)

    def add_commits_command(self):
        help_str = 'detect refactoring from commits in the csv'
        subparser = self.subparsers.add_parser('commits', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)

        help_str = 'comma separated commits list. please write a_commit hash and b_commit hash per line'
        subparser.add_argument('commits_list', help=help_str)
        subparser.set_defaults(func=self.detect_from_commits_list)

    def detect_from_commits_list(self, args):
        historage = Repo(args.historage_dir)
        extract_method_information = []
        try:
            for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
                a_commit = historage.commit(a_commit_hash)
                b_commit = historage.commit(b_commit_hash)
                extract_method_information.extend(detect_pullup_method_from_commit(a_commit, b_commit))
        except ValueError:
            print "Invalid input."
            return
        except BadObject, name:
            print "Invalid hash of the commit:", name.message

        for a_commit, b_commit, org_commit, a_package, b_package, c, m, method, sim in extract_method_information:
            print self.format_for_umldiff('jedit', a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                                          sim)

def main():
    parser = RefactoringDetectionCommandParser()
    parser.parse_and_execute_command()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from git.repo import Repo
from gitdb.exc import BadObject
from kenja.detection.pull_up_method import detect_pullup_method_from_commit
from kenja.detection.pull_up_method import detect_pull_up_method
import argparse
import csv

class RefactoringDetectionCommandParser:
    def __init__(self):
        self.parser = argparse.ArgumentParser(description='Kenja a refactoring detection tool')
        self.subparsers = self.parser.add_subparsers()

        self.add_all_command()
        self.add_commits_command()

    def parse_and_execute_command(self):
        args = self.parser.parse_args()
        args.func(args)

    def add_all_command(self):
        help_str = 'detect Pull Up Method refactoring from all commits in the Historage'
        subparser = self.subparsers.add_parser('all', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)
        subparser.set_defaults(func=self.detect_all)

    def detect_all(self, args):
        historage = Repo(args.historage_dir)
        pull_up_method_candidates = detect_pull_up_method(historage)
        print '"old_commit","new_commit","old_org_commit","new_org_commit",' \
            + '"src_method","dst_method","similarity","isSamePrameters"'
        for info in pull_up_method_candidates:
            # info[6] = sim
            info[6] = str(info[6])
            print ','.join(['"' + s + '"' for s in info])

        # candidate_revisions = set()
        # for a_commit, b_commit, org_commit, a_package, b_package, c, m, method, sim in extract_method_information:
        #     candidate_revisions.add(b_commit)
        #     print self.format_for_umldiff('jedit', a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
        #                                   sim)

        # print 'candidates:', len(extract_method_information)
        # print 'candidate revisions:', len(candidate_revisions)

    def format_for_umldiff(self, package_prefix, a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                           sim):
        target_method_info = [package_prefix]
        if a_package:
            target_method_info.append(a_package)
        target_method_info.extend((c, m))
        target_method = '.'.join(target_method_info)
        extracted_method_info = [package_prefix]
        if b_package:
            extracted_method_info.append(b_package)
        extracted_method_info.extend((c, method))
        extracted_method = '.'.join(extracted_method_info)
        return '"%s","%s","%s","%s","%s","%s"' % (a_commit, b_commit, org_commit, target_method, extracted_method, sim)

    def add_commits_command(self):
        help_str = 'detect refactoring from commits in the csv'
        subparser = self.subparsers.add_parser('commits', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)

        help_str = 'comma separated commits list. please write a_commit hash and b_commit hash per line'
        subparser.add_argument('commits_list', help=help_str)
        subparser.set_defaults(func=self.detect_from_commits_list)

    def detect_from_commits_list(self, args):
        historage = Repo(args.historage_dir)
        extract_method_information = []
        try:
            for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
                a_commit = historage.commit(a_commit_hash)
                b_commit = historage.commit(b_commit_hash)
                extract_method_information.extend(detect_pullup_method_from_commit(a_commit, b_commit))
        except ValueError:
            print "Invalid input."
            return
        except BadObject, name:
            print "Invalid hash of the commit:", name.message

        for a_commit, b_commit, org_commit, a_package, b_package, c, m, method, sim in extract_method_information:
            print self.format_for_umldiff('jedit', a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                                          sim)

def main():
    parser = RefactoringDetectionCommandParser()
    parser.parse_and_execute_command()

if __name__ == '__main__':
    main()
<commit_msg>removed old commneted code<commit_after>from __future__ import absolute_import
from git.repo import Repo
from gitdb.exc import BadObject
from kenja.detection.pull_up_method import detect_pullup_method_from_commit
from kenja.detection.pull_up_method import detect_pull_up_method
import argparse
import csv

class RefactoringDetectionCommandParser:
    def __init__(self):
        self.parser = argparse.ArgumentParser(description='Kenja a refactoring detection tool')
        self.subparsers = self.parser.add_subparsers()

        self.add_all_command()
        self.add_commits_command()

    def parse_and_execute_command(self):
        args = self.parser.parse_args()
        args.func(args)

    def add_all_command(self):
        help_str = 'detect Pull Up Method refactoring from all commits in the Historage'
        subparser = self.subparsers.add_parser('all', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)
        subparser.set_defaults(func=self.detect_all)

    def detect_all(self, args):
        historage = Repo(args.historage_dir)
        pull_up_method_candidates = detect_pull_up_method(historage)
        print '"old_commit","new_commit","old_org_commit","new_org_commit",' \
            + '"src_method","dst_method","similarity","isSamePrameters"'
        for info in pull_up_method_candidates:
            # info[6] = sim
            info[6] = str(info[6])
            print ','.join(['"' + s + '"' for s in info])

    def format_for_umldiff(self, package_prefix, a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                           sim):
        target_method_info = [package_prefix]
        if a_package:
            target_method_info.append(a_package)
        target_method_info.extend((c, m))
        target_method = '.'.join(target_method_info)
        extracted_method_info = [package_prefix]
        if b_package:
            extracted_method_info.append(b_package)
        extracted_method_info.extend((c, method))
        extracted_method = '.'.join(extracted_method_info)
        return '"%s","%s","%s","%s","%s","%s"' % (a_commit, b_commit, org_commit, target_method, extracted_method, sim)

    def add_commits_command(self):
        help_str = 'detect refactoring from commits in the csv'
        subparser = self.subparsers.add_parser('commits', help=help_str)

        help_str = 'path of Historage dir'
        subparser.add_argument('historage_dir', help=help_str)

        help_str = 'comma separated commits list. please write a_commit hash and b_commit hash per line'
        subparser.add_argument('commits_list', help=help_str)
        subparser.set_defaults(func=self.detect_from_commits_list)

    def detect_from_commits_list(self, args):
        historage = Repo(args.historage_dir)
        extract_method_information = []
        try:
            for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
                a_commit = historage.commit(a_commit_hash)
                b_commit = historage.commit(b_commit_hash)
                extract_method_information.extend(detect_pullup_method_from_commit(a_commit, b_commit))
        except ValueError:
            print "Invalid input."
            return
        except BadObject, name:
            print "Invalid hash of the commit:", name.message

        for a_commit, b_commit, org_commit, a_package, b_package, c, m, method, sim in extract_method_information:
            print self.format_for_umldiff('jedit', a_commit, b_commit, org_commit, a_package, b_package, c, m, method,
                                          sim)

def main():
    parser = RefactoringDetectionCommandParser()
    parser.parse_and_execute_command()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Tests for passing invalid endpoints which require a target ID to be given.
"""

import uuid
from urllib.parse import urljoin

import pytest
import requests
from requests import codes
from requests_mock import GET

from common.constants import ResultCodes
from tests.conftest import VuforiaServerCredentials
from tests.mock_vws.utils import assert_vws_failure
from vws._request_utils import authorization_header, rfc_1123_date

class Endpoint:
    """
    XXX
    """

    def __init__(self, path: str, method: int) -> None:
        """
        """
        self.path = path
        self.method = method

# ENDPOINTS = [
#     endpoint(path='/targets', method=get),
#     endpoint(path='/duplicates', method=get),
# ]

@pytest.fixture()
def target_list():
    return Endpoint(path='/targets', method=GET)

@pytest.fixture()
def get_duplicates():
    return Endpoint(path='/duplicates', method=GET)

@pytest.fixture(params=['target_list', 'get_duplicates'])
def endpoint(request):
    return request.getfixturevalue(request.param)

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestInvalidGivenID:
    """
    Tests for giving an invalid ID to endpoints which require a target ID to
    be given.
    """

    def test_not_real_id(self,
                         vuforia_server_credentials: VuforiaServerCredentials,
                         endpoint,
                         ) -> None:
        """
        A `NOT_FOUND` error is returned when an endpoint is given a target ID
        of a target which does not exist.
        """
        request_path = endpoint.path + '/' + uuid.uuid4().hex
        date = rfc_1123_date()

        authorization_string = authorization_header(
            access_key=vuforia_server_credentials.access_key,
            secret_key=vuforia_server_credentials.secret_key,
            method=GET,
            content=b'',
            content_type='',
            date=date,
            request_path=request_path,
        )

        headers = {
            "Authorization": authorization_string,
            "Date": date,
        }

        url = urljoin('https://vws.vuforia.com/', request_path)
        response = requests.request(
            method=GET,
            url=url,
            headers=headers,
            data=b'',
        )

        assert_vws_failure(
            response=response,
            status_code=codes.NOT_FOUND,
            result_code=ResultCodes.UNKNOWN_TARGET,
        )
<commit_msg>Start cleanup<commit_after>"""
Tests for passing invalid endpoints which require a target ID to be given.
"""

import uuid
from urllib.parse import urljoin

import pytest
import requests
from requests import codes
from requests_mock import GET

from common.constants import ResultCodes
from tests.conftest import VuforiaServerCredentials
from tests.mock_vws.utils import assert_vws_failure
from vws._request_utils import authorization_header, rfc_1123_date

class Endpoint:
    """
    XXX
    """

    def __init__(self, path: str, method: int) -> None:
        """
        """
        self.path = path
        self.method = method

@pytest.fixture()
def target_list():
    return Endpoint(path='/targets', method=GET)

@pytest.fixture()
def get_duplicates():
    return Endpoint(path='/duplicates', method=GET)

@pytest.fixture(params=['target_list', 'get_duplicates'])
def endpoint(request):
    return request.getfixturevalue(request.param)

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestInvalidGivenID:
    """
    Tests for giving an invalid ID to endpoints which require a target ID to
    be given.
    """

    def test_not_real_id(self,
                         vuforia_server_credentials: VuforiaServerCredentials,
                         endpoint: Endpoint,
                         ) -> None:
        """
        A `NOT_FOUND` error is returned when an endpoint is given a target ID
        of a target which does not exist.
        """
        request_path = endpoint.path + '/' + uuid.uuid4().hex
        date = rfc_1123_date()

        authorization_string = authorization_header(
            access_key=vuforia_server_credentials.access_key,
            secret_key=vuforia_server_credentials.secret_key,
            method=GET,
            content=b'',
            content_type='',
            date=date,
            request_path=request_path,
        )

        headers = {
            "Authorization": authorization_string,
            "Date": date,
        }

        url = urljoin('https://vws.vuforia.com/', request_path)
        response = requests.request(
            method=GET,
            url=url,
            headers=headers,
            data=b'',
        )

        assert_vws_failure(
            response=response,
            status_code=codes.NOT_FOUND,
            result_code=ResultCodes.UNKNOWN_TARGET,
        )
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

import cv2
import object_detector.utils as utils
import object_detector.file_io as file_io
import numpy as np
import random

import numpy as np
import os

class FeatureExtractor():

    def __init__(self, descriptor, patch_size, data_file):
        self._desc = descriptor
        self._patch_size = patch_size

        if data_file is None:
            self._dataset = []
        else:
            self._dataset = file_io.FileHDF5().read(data_file, "label_and_features").tolist()

    # Todo : Template Method Pattern??
    def add_positive_sets(self, image_dir, pattern, annotation_path, sample_ratio=1.0, padding=5, augment=True, label=1):

        features_set = []
        image_files = self._get_image_files(image_dir, pattern, sample_ratio)

        for image_file in image_files:
            image = cv2.imread(image_file)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
            image_id = utils.get_file_id(image_file)

            annotation_file = "{}/annotation_{}.mat".format(annotation_path, image_id)
            bb = file_io.FileMat().read(annotation_file)["box_coord"][0]
            roi = utils.crop_bb(image, bb, padding=padding, dst_size=self._patch_size)

            patches = (roi, cv2.flip(roi, 1)) if augment else (roi,)

            # Todo : augment modulization
            features = self._desc.describe(patches)
            features_set += features.tolist()

        labels = np.zeros((len(features_set), 1)) + label
        dataset = np.concatenate([labels, np.array(features_set)], axis=1)
        self._dataset += dataset.tolist()

    def add_negative_sets(self, image_dir, pattern, n_samples_per_img, sample_ratio=1.0):
        # Todo : progressbar
        # Todo : 한꺼번에 write 하지말고, 10개 이미지를 모아서 write 를 자주하는 방식으로 수정

        features_set = []
        image_files = self._get_image_files(image_dir, pattern, sample_ratio)

        for image_file in image_files:
            image = cv2.imread(image_file)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = utils.crop_random(image, self._patch_size, n_samples_per_img)

            features = self._desc.describe(patches)
            features_set += features.tolist()

        labels = np.zeros((len(features_set), 1))
        dataset = np.concatenate([labels, np.array(features_set)], axis=1)
        self._dataset += dataset.tolist()

    def add_data(self, features, label):
        labels = np.zeros((len(features), 1)) + label
        dataset = np.concatenate([labels, features], axis=1)

        self._dataset += dataset.tolist()

    def save(self, data_file):
        file_io.FileHDF5().write(np.array(self._dataset), data_file, "label_and_features")

    def summary(self):

        labels = np.array(self._dataset)[:, 0]
        feature_shape = np.array(self._dataset)[:, 1:].shape

        n_positive_samples = len(labels[labels > 0])
        n_negative_samples = len(labels[labels == 0])
        n_hard_negative_samples = len(labels[labels == -1])

        print "[FeatureGetter INFO] Positive samples: {}, Negative samples: {}, Hard Negative Mined samples: {}".format(n_positive_samples, n_negative_samples, n_hard_negative_samples)
        print "[FeatureGetter INFO] Feature Dimension: {}".format(feature_shape[1])

    def get_dataset(self, include_hard_negative=True):
        if self._dataset is None:
            raise ValueError('There is no dataset in this instance')
        else:
            dataset = np.array(self._dataset)
            if include_hard_negative:
                dataset[dataset[:,0] < 0, 0] = 0
            else:
                dataset = dataset[dataset[:,0] >= 0]
            return dataset

    def _get_image_files(self, directory, pattern, sample_ratio):
        image_files = file_io.list_files(directory, pattern)
        image_files = random.sample(image_files, int(len(image_files) * sample_ratio))
        return image_files

class SVHNFeatureExtractor(FeatureExtractor):
    # Todo : Template Method Pattern??
    def add_positive_sets(self, annotation_file, sample_ratio=1.0, padding=5, augment=True, label=1):

        features_set = []
        labels_set = []
        image_path = os.path.split(annotation_file)[0]
        annotations = file_io.FileJson().read(annotation_file)
        annotations = annotations[:int(len(annotations)*sample_ratio)]

        for annotation in annotations:
            image = cv2.imread(os.path.join(image_path, annotation["filename"]))
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = []
            labels = []
            for box in annotation["boxes"]:
                x1 = int(box["left"])
                y1 = int(box["top"])
                w = int(box["width"])
                h = int(box["height"])

                bb = (y1, y1+h, x1, x1+w)
                label = int(box["label"])

                roi = utils.crop_bb(image, bb, padding=padding, dst_size=self._patch_size)
                patches.append(roi)
                labels.append(label)

            # Todo : augment modulization
            features = self._desc.describe(patches)
            features_set += features.tolist()
            labels_set += labels

        #labels = np.zeros((len(features_set), 1)) + label
        dataset = np.concatenate([np.array(labels_set).reshape(-1, 1), np.array(features_set)], axis=1)
        self._dataset += dataset.tolist()

import object_detector.descriptor as desc
import object_detector.file_io as file_io

def setup_extractor():
    descriptor = desc.HOG(9, [4,4], [2,2])
    extractor = SVHNFeatureExtractor(descriptor, [32, 16], None)
    return extractor

def setup_params():
    annotation_filename = "../datasets/positive/digitStruct.json"
    negative_dir = "../datasets/negative"
    output_file = "svhn_features.hdf5"
    return annotation_filename, negative_dir, output_file

def test_add_positive_behavior():

    extractor = setup_extractor()
    annotation_filename, negative_dir, output_file = setup_params()

    # 2. Get Feature sets
    extractor.add_positive_sets(annotation_file=annotation_filename,
                             sample_ratio=1.0,
                             padding=0,
                             )

    dataset = extractor.get_dataset(include_hard_negative=True)
    assert dataset.shape == (8, 757)

def test_add_negative_behavior():

    extractor = setup_extractor()
    annotation_filename, negative_dir, output_file = setup_params()

    # Todo : positive sample 숫자에 따라 negative sample 숫자를 자동으로 정할 수 있도록 설정
    extractor.add_negative_sets(image_dir=negative_dir,
                             pattern="*.jpg",
                             n_samples_per_img=10,
                             sample_ratio=1.0)

    dataset = extractor.get_dataset(include_hard_negative=True)
    assert dataset.shape == (40, 757)

#     # 3. Save dataset
#     extractor.save(data_file=output_file)

if __name__ == "__main__":
    import pytest
    pytest.main([__file__])

<commit_msg>features와 labels 를 따로 저장하도록 리팩토링 <commit_after>#-*- coding: utf-8 -*-

import cv2
import object_detector.utils as utils
import object_detector.file_io as file_io
import numpy as np
import random

import os

class FeatureExtractor():

    def __init__(self, descriptor, patch_size, data_file):
        self._desc = descriptor
        self._patch_size = patch_size

        if data_file is None:
            self._features = None   # (N, n, m, ...)
            self._labels = None     # (N, 1)
#         else:
#             self._images = None
#             self._labels = None
#
#             # self._dataset = file_io.FileHDF5().read(data_file, "label_and_features").tolist()

    # Todo : Template Method Pattern??
    def add_positive_sets(self, image_dir, pattern, annotation_path, sample_ratio=1.0, padding=5, augment=True, label=1):

        features_set = []
        image_files = self._get_image_files(image_dir, pattern, sample_ratio)

        for image_file in image_files:
            image = cv2.imread(image_file)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
            image_id = utils.get_file_id(image_file)

            annotation_file = "{}/annotation_{}.mat".format(annotation_path, image_id)
            bb = file_io.FileMat().read(annotation_file)["box_coord"][0]
            roi = utils.crop_bb(image, bb, padding=padding, dst_size=self._patch_size)

            patches = (roi, cv2.flip(roi, 1)) if augment else (roi,)

            # Todo : augment modulization
            features = self._desc.describe(patches)
            features_set += features.tolist()

        self._labels = np.zeros((len(features_set), 1)) + label
        self._features = np.array(features_set)

    def add_negative_sets(self, image_dir, pattern, n_samples_per_img, sample_ratio=1.0):
        # Todo : progressbar
        # Todo : 한꺼번에 write 하지말고, 10개 이미지를 모아서 write 를 자주하는 방식으로 수정

        features_set = []
        image_files = self._get_image_files(image_dir, pattern, sample_ratio)

        for image_file in image_files:
            image = cv2.imread(image_file)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = utils.crop_random(image, self._patch_size, n_samples_per_img)

            features = self._desc.describe(patches)
            features_set += features.tolist()

        self._labels = np.zeros((len(features_set), 1))
        self._features = np.array(features_set)

    def add_data(self, features, label):
        labels = np.zeros((len(features), 1)) + label

        if self._features is None:
            self._features = features
            self._labels = label
        else:
            self._features = np.concatenate([self._features, features], axis=0)
            self._labels = np.concatenate([self._labels, labels], axis=0)

    def save(self, data_file):
        file_io.FileHDF5().write(self._features, data_file, "features")
        file_io.FileHDF5().write(self._labels, data_file, "labels")

    def summary(self):

        labels = self._labels
        feature_shape = self._features.shape

        n_positive_samples = len(labels[labels > 0])
        n_negative_samples = len(labels[labels == 0])
        n_hard_negative_samples = len(labels[labels == -1])

        print "[FeatureGetter INFO] Positive samples: {}, Negative samples: {}, Hard Negative Mined samples: {}".format(n_positive_samples, n_negative_samples, n_hard_negative_samples)
        print "[FeatureGetter INFO] Feature Dimension: {}".format(feature_shape[1])

    def get_dataset(self, include_hard_negative=True):

        if self._features is None:
            raise ValueError('There is no dataset in this instance')
        else:
            labels = self._labels
            if include_hard_negative:
                features = self._features
                labels[labels < 0] = 0
            else:
                features = self._features[self._labels >= 0]
                labels = self._labels[self._labels >= 0]
            return features, labels

    def _get_image_files(self, directory, pattern, sample_ratio):
        image_files = file_io.list_files(directory, pattern)
        image_files = random.sample(image_files, int(len(image_files) * sample_ratio))
        return image_files

class SVHNFeatureExtractor(FeatureExtractor):
    # Todo : Template Method Pattern??
    def add_positive_sets(self, annotation_file, sample_ratio=1.0, padding=5, augment=True, label=1):

        features_set = []
        labels_set = []
        image_path = os.path.split(annotation_file)[0]
        annotations = file_io.FileJson().read(annotation_file)
        annotations = annotations[:int(len(annotations)*sample_ratio)]

        for annotation in annotations:
            image = cv2.imread(os.path.join(image_path, annotation["filename"]))
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = []
            labels = []
            for box in annotation["boxes"]:
                x1 = int(box["left"])
                y1 = int(box["top"])
                w = int(box["width"])
                h = int(box["height"])

                bb = (y1, y1+h, x1, x1+w)
                label = int(box["label"])

                roi = utils.crop_bb(image, bb, padding=padding, dst_size=self._patch_size)
                patches.append(roi)
                labels.append(label)

            # Todo : augment modulization
            features = self._desc.describe(patches)
            features_set += features.tolist()
            labels_set += labels

        self._features = np.array(features_set)
        self._labels = np.array(labels_set).reshape(-1, 1)

import object_detector.descriptor as desc

def setup_extractor():
    descriptor = desc.HOG(9, [4,4], [2,2])
    extractor = SVHNFeatureExtractor(descriptor, [32, 16], None)
    return extractor

def setup_params():
    annotation_filename = "../datasets/positive/digitStruct.json"
    negative_dir = "../datasets/negative"
    output_file = "svhn_features.hdf5"
    return annotation_filename, negative_dir, output_file

def test_add_positive_behavior():

    extractor = setup_extractor()
    annotation_filename, negative_dir, output_file = setup_params()

    # 2. Get Feature sets
    extractor.add_positive_sets(annotation_file=annotation_filename,
                             sample_ratio=1.0,
                             padding=0,
                             )

    features, labels = extractor.get_dataset(include_hard_negative=True)
    assert features.shape == (8, 756)
    assert labels.shape == (8, 1)

def test_add_negative_behavior():

    extractor = setup_extractor()
    annotation_filename, negative_dir, output_file = setup_params()

    # Todo : positive sample 숫자에 따라 negative sample 숫자를 자동으로 정할 수 있도록 설정
    extractor.add_negative_sets(image_dir=negative_dir,
                             pattern="*.jpg",
                             n_samples_per_img=10,
                             sample_ratio=1.0)

    features, labels = extractor.get_dataset(include_hard_negative=True)
    assert features.shape == (40, 756)
    assert labels.shape == (40, 1)

#     # 3. Save dataset
#     extractor.save(data_file=output_file)

if __name__ == "__main__":
    import pytest
    pytest.main([__file__])

<|endoftext|>


<commit_before>from sqlalchemy_continuum.plugins import PluginCollection

class TestPluginCollection(object):
    def test_init(self):
        assert PluginCollection([1, 2, 3]).plugins == [1, 2, 3]

    def test_iter(self):
        assert list(PluginCollection([1, 2, 3])) == [1, 2, 3]

    def test_len(self):
        assert len(PluginCollection()) == 0
        assert len(PluginCollection([1, 2, 3])) == 3

    def test_getitem(self):
        assert PluginCollection([1, 2])[0] == 1

    def test_setitem(self):
        coll = PluginCollection([1, 2])
        coll[0] = 2
        assert coll[0] == 2

    def test_delitem(self):
        coll = PluginCollection([1, 2])
        del coll[0]
        assert list(coll) == [2]

    def test_append(self):
        coll = PluginCollection([1, 2])
        coll.append(3)
        assert list(coll) == [1, 2, 3]
<commit_msg>Add tests for getattr<commit_after>from sqlalchemy_continuum.plugins import PluginCollection

class TestPluginCollection(object):
    def test_init(self):
        assert PluginCollection([1, 2, 3]).plugins == [1, 2, 3]

    def test_iter(self):
        assert list(PluginCollection([1, 2, 3])) == [1, 2, 3]

    def test_len(self):
        assert len(PluginCollection()) == 0
        assert len(PluginCollection([1, 2, 3])) == 3

    def test_getitem(self):
        assert PluginCollection([1, 2])[0] == 1

    def test_setitem(self):
        coll = PluginCollection([1, 2])
        coll[0] = 2
        assert coll[0] == 2

    def test_delitem(self):
        coll = PluginCollection([1, 2])
        del coll[0]
        assert list(coll) == [2]

    def test_append(self):
        coll = PluginCollection([1, 2])
        coll.append(3)
        assert list(coll) == [1, 2, 3]

    def test_getattr(self):
        class MyPlugin(object):
            def some_action(self):
                return 4
        coll = PluginCollection([MyPlugin(), MyPlugin()])
        assert list(coll.some_action()) == [4, 4]
<|endoftext|>


<commit_before>"""
Various complex queries that have been problematic in the past.
"""

import threading

from django.db import models

class DumbCategory(models.Model):
    pass

class NamedCategory(DumbCategory):
    name = models.CharField(max_length=10)

class Tag(models.Model):
    name = models.CharField(max_length=10)
    parent = models.ForeignKey('self', blank=True, null=True,
            related_name='children')
    category = models.ForeignKey(NamedCategory, null=True, default=None)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class Note(models.Model):
    note = models.CharField(max_length=100)
    misc = models.CharField(max_length=10)

    class Meta:
        ordering = ['note']

    def __unicode__(self):
        return self.note

    def __init__(self, *args, **kwargs):
        super(Note, self).__init__(*args, **kwargs)
        # Regression for #13227 -- having an attribute that
        # is unpickleable doesn't stop you from cloning queries
        # that use objects of that type as an argument.
        self.lock = threading.Lock()

class Annotation(models.Model):
    name = models.CharField(max_length=10)
    tag = models.ForeignKey(Tag)
    notes = models.ManyToManyField(Note)

    def __unicode__(self):
        return self.name

class ExtraInfo(models.Model):
    info = models.CharField(max_length=100)
    note = models.ForeignKey(Note)

    class Meta:
        ordering = ['info']

    def __unicode__(self):
        return self.info

class Author(models.Model):
    name = models.CharField(max_length=10)
    num = models.IntegerField(unique=True)
    extra = models.ForeignKey(ExtraInfo)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class Item(models.Model):
    name = models.CharField(max_length=10)
    created = models.DateTimeField()
    modified = models.DateTimeField(blank=True, null=True)
    tags = models.ManyToManyField(Tag, blank=True, null=True)
    creator = models.ForeignKey(Author)
    note = models.ForeignKey(Note)

    class Meta:
        ordering = ['-note', 'name']

    def __unicode__(self):
        return self.name

class Report(models.Model):
    name = models.CharField(max_length=10)
    creator = models.ForeignKey(Author, to_field='num', null=True)

    def __unicode__(self):
        return self.name

class Ranking(models.Model):
    rank = models.IntegerField()
    author = models.ForeignKey(Author)

    class Meta:
        # A complex ordering specification. Should stress the system a bit.
        ordering = ('author__extra__note', 'author__name', 'rank')

    def __unicode__(self):
        return '%d: %s' % (self.rank, self.author.name)

class Cover(models.Model):
    title = models.CharField(max_length=50)
    item = models.ForeignKey(Item)

    class Meta:
        ordering = ['item']

    def __unicode__(self):
        return self.title

class Number(models.Model):
    num = models.IntegerField()

    def __unicode__(self):
        return unicode(self.num)

# Symmetrical m2m field with a normal field using the reverse accesor name
# ("valid").
class Valid(models.Model):
    valid = models.CharField(max_length=10)
    parent = models.ManyToManyField('self')

    class Meta:
        ordering = ['valid']

# Some funky cross-linked models for testing a couple of infinite recursion
# cases.
class X(models.Model):
    y = models.ForeignKey('Y')

class Y(models.Model):
    x1 = models.ForeignKey(X, related_name='y1')

# Some models with a cycle in the default ordering. This would be bad if we
# didn't catch the infinite loop.
class LoopX(models.Model):
    y = models.ForeignKey('LoopY')

    class Meta:
        ordering = ['y']

class LoopY(models.Model):
    x = models.ForeignKey(LoopX)

    class Meta:
        ordering = ['x']

class LoopZ(models.Model):
    z = models.ForeignKey('self')

    class Meta:
        ordering = ['z']

# A model and custom default manager combination.
class CustomManager(models.Manager):
    def get_query_set(self):
        qs = super(CustomManager, self).get_query_set()
        return qs.filter(public=True, tag__name='t1')

class ManagedModel(models.Model):
    data = models.CharField(max_length=10)
    tag = models.ForeignKey(Tag)
    public = models.BooleanField(default=True)

    objects = CustomManager()
    normal_manager = models.Manager()

    def __unicode__(self):
        return self.data

# An inter-related setup with multiple paths from Child to Detail.
class Detail(models.Model):
    data = models.CharField(max_length=10)

class MemberManager(models.Manager):
    def get_query_set(self):
        return super(MemberManager, self).get_query_set().select_related("details")

class Member(models.Model):
    name = models.CharField(max_length=10)
    details = models.OneToOneField(Detail, primary_key=True)

    objects = MemberManager()

class Child(models.Model):
    person = models.OneToOneField(Member, primary_key=True)
    parent = models.ForeignKey(Member, related_name="children")

# Custom primary keys interfered with ordering in the past.
class CustomPk(models.Model):
    name = models.CharField(max_length=10, primary_key=True)
    extra = models.CharField(max_length=10)

    class Meta:
        ordering = ['name', 'extra']

class Related(models.Model):
    custom = models.ForeignKey(CustomPk)

# An inter-related setup with a model subclass that has a nullable
# path to another model, and a return path from that model.

class Celebrity(models.Model):
    name = models.CharField("Name", max_length=20)
    greatest_fan = models.ForeignKey("Fan", null=True, unique=True)

class TvChef(Celebrity):
    pass

class Fan(models.Model):
    fan_of = models.ForeignKey(Celebrity)

# Multiple foreign keys
class LeafA(models.Model):
    data = models.CharField(max_length=10)

    def __unicode__(self):
        return self.data

class LeafB(models.Model):
    data = models.CharField(max_length=10)

class Join(models.Model):
    a = models.ForeignKey(LeafA)
    b = models.ForeignKey(LeafB)

class ReservedName(models.Model):
    name = models.CharField(max_length=20)
    order = models.IntegerField()

    def __unicode__(self):
        return self.name

# A simpler shared-foreign-key setup that can expose some problems.
class SharedConnection(models.Model):
    data = models.CharField(max_length=10)

class PointerA(models.Model):
    connection = models.ForeignKey(SharedConnection)

class PointerB(models.Model):
    connection = models.ForeignKey(SharedConnection)

# Multi-layer ordering
class SingleObject(models.Model):
    name = models.CharField(max_length=10)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class RelatedObject(models.Model):
    single = models.ForeignKey(SingleObject)

    class Meta:
        ordering = ['single']

class Plaything(models.Model):
    name = models.CharField(max_length=10)
    others = models.ForeignKey(RelatedObject, null=True)

    class Meta:
        ordering = ['others']

    def __unicode__(self):
        return self.name

class Article(models.Model):
    name = models.CharField(max_length=20)
    created = models.DateTimeField()

class Food(models.Model):
    name = models.CharField(max_length=20, unique=True)

    def __unicode__(self):
        return self.name

class Eaten(models.Model):
    food = models.ForeignKey(Food, to_field="name")
    meal = models.CharField(max_length=20)

    def __unicode__(self):
        return u"%s at %s" % (self.food, self.meal)

class Node(models.Model):
    num = models.IntegerField(unique=True)
    parent = models.ForeignKey("self", to_field="num", null=True)

    def __unicode__(self):
        return u"%s" % self.num

# Bug #12252
class ObjectA(models.Model):
    name = models.CharField(max_length=50)

    def __unicode__(self):
        return self.name

class ObjectB(models.Model):
    name = models.CharField(max_length=50)
    objecta = models.ForeignKey(ObjectA)
    num = models.PositiveSmallIntegerField()

    def __unicode__(self):
        return self.name

class ObjectC(models.Model):
    name = models.CharField(max_length=50)
    objecta = models.ForeignKey(ObjectA)
    objectb = models.ForeignKey(ObjectB)

    def __unicode__(self):
       return self.name

class SimpleCategory(models.Model):
    name = models.CharField(max_length=10)

    def __unicode__(self):
        return self.name

class SpecialCategory(SimpleCategory):
    special_name = models.CharField(max_length=10)

    def __unicode__(self):
        return self.name + " " + self.special_name

class CategoryItem(models.Model):
    category = models.ForeignKey(SimpleCategory)

    def __unicode__(self):
 	    return "category item: " + str(self.category)

class OneToOneCategory(models.Model):
    new_name = models.CharField(max_length=10)
    category = models.OneToOneField(SimpleCategory)

    def __unicode__(self):
        return "one2one " + self.new_name
    <commit_msg>Fixed a broken test from r16656.<commit_after>"""
Various complex queries that have been problematic in the past.
"""

import threading

from django.db import models

class DumbCategory(models.Model):
    pass

class NamedCategory(DumbCategory):
    name = models.CharField(max_length=10)

class Tag(models.Model):
    name = models.CharField(max_length=10)
    parent = models.ForeignKey('self', blank=True, null=True,
            related_name='children')
    category = models.ForeignKey(NamedCategory, null=True, default=None)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class Note(models.Model):
    note = models.CharField(max_length=100)
    misc = models.CharField(max_length=10)

    class Meta:
        ordering = ['note']

    def __unicode__(self):
        return self.note

    def __init__(self, *args, **kwargs):
        super(Note, self).__init__(*args, **kwargs)
        # Regression for #13227 -- having an attribute that
        # is unpickleable doesn't stop you from cloning queries
        # that use objects of that type as an argument.
        self.lock = threading.Lock()

class Annotation(models.Model):
    name = models.CharField(max_length=10)
    tag = models.ForeignKey(Tag)
    notes = models.ManyToManyField(Note)

    def __unicode__(self):
        return self.name

class ExtraInfo(models.Model):
    info = models.CharField(max_length=100)
    note = models.ForeignKey(Note)

    class Meta:
        ordering = ['info']

    def __unicode__(self):
        return self.info

class Author(models.Model):
    name = models.CharField(max_length=10)
    num = models.IntegerField(unique=True)
    extra = models.ForeignKey(ExtraInfo)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class Item(models.Model):
    name = models.CharField(max_length=10)
    created = models.DateTimeField()
    modified = models.DateTimeField(blank=True, null=True)
    tags = models.ManyToManyField(Tag, blank=True, null=True)
    creator = models.ForeignKey(Author)
    note = models.ForeignKey(Note)

    class Meta:
        ordering = ['-note', 'name']

    def __unicode__(self):
        return self.name

class Report(models.Model):
    name = models.CharField(max_length=10)
    creator = models.ForeignKey(Author, to_field='num', null=True)

    def __unicode__(self):
        return self.name

class Ranking(models.Model):
    rank = models.IntegerField()
    author = models.ForeignKey(Author)

    class Meta:
        # A complex ordering specification. Should stress the system a bit.
        ordering = ('author__extra__note', 'author__name', 'rank')

    def __unicode__(self):
        return '%d: %s' % (self.rank, self.author.name)

class Cover(models.Model):
    title = models.CharField(max_length=50)
    item = models.ForeignKey(Item)

    class Meta:
        ordering = ['item']

    def __unicode__(self):
        return self.title

class Number(models.Model):
    num = models.IntegerField()

    def __unicode__(self):
        return unicode(self.num)

# Symmetrical m2m field with a normal field using the reverse accesor name
# ("valid").
class Valid(models.Model):
    valid = models.CharField(max_length=10)
    parent = models.ManyToManyField('self')

    class Meta:
        ordering = ['valid']

# Some funky cross-linked models for testing a couple of infinite recursion
# cases.
class X(models.Model):
    y = models.ForeignKey('Y')

class Y(models.Model):
    x1 = models.ForeignKey(X, related_name='y1')

# Some models with a cycle in the default ordering. This would be bad if we
# didn't catch the infinite loop.
class LoopX(models.Model):
    y = models.ForeignKey('LoopY')

    class Meta:
        ordering = ['y']

class LoopY(models.Model):
    x = models.ForeignKey(LoopX)

    class Meta:
        ordering = ['x']

class LoopZ(models.Model):
    z = models.ForeignKey('self')

    class Meta:
        ordering = ['z']

# A model and custom default manager combination.
class CustomManager(models.Manager):
    def get_query_set(self):
        qs = super(CustomManager, self).get_query_set()
        return qs.filter(public=True, tag__name='t1')

class ManagedModel(models.Model):
    data = models.CharField(max_length=10)
    tag = models.ForeignKey(Tag)
    public = models.BooleanField(default=True)

    objects = CustomManager()
    normal_manager = models.Manager()

    def __unicode__(self):
        return self.data

# An inter-related setup with multiple paths from Child to Detail.
class Detail(models.Model):
    data = models.CharField(max_length=10)

class MemberManager(models.Manager):
    def get_query_set(self):
        return super(MemberManager, self).get_query_set().select_related("details")

class Member(models.Model):
    name = models.CharField(max_length=10)
    details = models.OneToOneField(Detail, primary_key=True)

    objects = MemberManager()

class Child(models.Model):
    person = models.OneToOneField(Member, primary_key=True)
    parent = models.ForeignKey(Member, related_name="children")

# Custom primary keys interfered with ordering in the past.
class CustomPk(models.Model):
    name = models.CharField(max_length=10, primary_key=True)
    extra = models.CharField(max_length=10)

    class Meta:
        ordering = ['name', 'extra']

class Related(models.Model):
    custom = models.ForeignKey(CustomPk)

# An inter-related setup with a model subclass that has a nullable
# path to another model, and a return path from that model.

class Celebrity(models.Model):
    name = models.CharField("Name", max_length=20)
    greatest_fan = models.ForeignKey("Fan", null=True, unique=True)

class TvChef(Celebrity):
    pass

class Fan(models.Model):
    fan_of = models.ForeignKey(Celebrity)

# Multiple foreign keys
class LeafA(models.Model):
    data = models.CharField(max_length=10)

    def __unicode__(self):
        return self.data

class LeafB(models.Model):
    data = models.CharField(max_length=10)

class Join(models.Model):
    a = models.ForeignKey(LeafA)
    b = models.ForeignKey(LeafB)

class ReservedName(models.Model):
    name = models.CharField(max_length=20)
    order = models.IntegerField()

    def __unicode__(self):
        return self.name

# A simpler shared-foreign-key setup that can expose some problems.
class SharedConnection(models.Model):
    data = models.CharField(max_length=10)

class PointerA(models.Model):
    connection = models.ForeignKey(SharedConnection)

class PointerB(models.Model):
    connection = models.ForeignKey(SharedConnection)

# Multi-layer ordering
class SingleObject(models.Model):
    name = models.CharField(max_length=10)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class RelatedObject(models.Model):
    single = models.ForeignKey(SingleObject)

    class Meta:
        ordering = ['single']

class Plaything(models.Model):
    name = models.CharField(max_length=10)
    others = models.ForeignKey(RelatedObject, null=True)

    class Meta:
        ordering = ['others']

    def __unicode__(self):
        return self.name

class Article(models.Model):
    name = models.CharField(max_length=20)
    created = models.DateTimeField()

class Food(models.Model):
    name = models.CharField(max_length=20, unique=True)

    def __unicode__(self):
        return self.name

class Eaten(models.Model):
    food = models.ForeignKey(Food, to_field="name")
    meal = models.CharField(max_length=20)

    def __unicode__(self):
        return u"%s at %s" % (self.food, self.meal)

class Node(models.Model):
    num = models.IntegerField(unique=True)
    parent = models.ForeignKey("self", to_field="num", null=True)

    def __unicode__(self):
        return u"%s" % self.num

# Bug #12252
class ObjectA(models.Model):
    name = models.CharField(max_length=50)

    def __unicode__(self):
        return self.name

class ObjectB(models.Model):
    name = models.CharField(max_length=50)
    objecta = models.ForeignKey(ObjectA)
    num = models.PositiveSmallIntegerField()

    def __unicode__(self):
        return self.name

class ObjectC(models.Model):
    name = models.CharField(max_length=50)
    objecta = models.ForeignKey(ObjectA)
    objectb = models.ForeignKey(ObjectB)

    def __unicode__(self):
       return self.name

class SimpleCategory(models.Model):
    name = models.CharField(max_length=15)

    def __unicode__(self):
        return self.name

class SpecialCategory(SimpleCategory):
    special_name = models.CharField(max_length=15)

    def __unicode__(self):
        return self.name + " " + self.special_name

class CategoryItem(models.Model):
    category = models.ForeignKey(SimpleCategory)

    def __unicode__(self):
 	    return "category item: " + str(self.category)

class OneToOneCategory(models.Model):
    new_name = models.CharField(max_length=15)
    category = models.OneToOneField(SimpleCategory)

    def __unicode__(self):
        return "one2one " + self.new_name

<|endoftext|>


<commit_before>"""
>>> from django.http import HttpRequest, HttpResponse
>>> print repr(HttpRequest())
<HttpRequest
GET:{},
POST:{},
COOKIES:{},
META:{}>

>>> from django.core.handlers.wsgi import WSGIRequest
>>> print repr(WSGIRequest({'PATH_INFO': 'bogus', 'REQUEST_METHOD': 'bogus'}))
<WSGIRequest
GET:<QueryDict: {}>,
POST:<QueryDict: {}>,
COOKIES:{},
META:{...}>

>>> from django.core.handlers.modpython import ModPythonRequest
>>> class FakeModPythonRequest(ModPythonRequest):
...    def __init__(self, *args, **kwargs):
...        super(FakeModPythonRequest, self).__init__(*args, **kwargs)
...        self._get = self._post = self._meta = self._cookies = {}
>>> class Dummy:
...     def get_options(self):
...         return {}
>>> req = Dummy()
>>> req.uri = 'bogus'
>>> print repr(FakeModPythonRequest(req))
<ModPythonRequest
path:bogus,
GET:{},
POST:{},
COOKIES:{},
META:{}>

>>> from django.http import parse_cookie
>>> parse_cookie('invalid:key=true')
{}

>>> request = HttpRequest()
>>> print request.build_absolute_uri(location="https://www.example.com/asdf")
https://www.example.com/asdf
>>> request.get_host = lambda: 'www.example.com'
>>> request.path = ''
>>> print request.build_absolute_uri(location="/path/with:colons")
http://www.example.com/path/with:colons

# Test cookie datetime expiration logic
>>> from datetime import datetime, timedelta
>>> delta = timedelta(seconds=10)
>>> response = HttpResponse()
>>> response.set_cookie('datetime', expires=datetime.utcnow()+delta)
>>> datetime_cookie = response.cookies['datetime']
>>> datetime_cookie['max-age']
10
>>> response.set_cookie('datetime', expires=datetime(2028, 1, 1, 4, 5, 6))
>>> response.cookies['datetime']['expires']
'Sat, 01-Jan-2028 04:05:06 GMT'

# Test automatically setting cookie expires if only max_age is provided
>>> response.set_cookie('max_age', max_age=10)
>>> max_age_cookie = response.cookies['max_age']
>>> max_age_cookie['max-age']
10
>>> from django.utils.http import cookie_date
>>> import time
>>> max_age_cookie['expires'] == cookie_date(time.time()+10)
True
"""
<commit_msg>Fixed #14351 -- Modified the requests test case to avoid an timing problem in the test.<commit_after>"""
>>> from django.http import HttpRequest, HttpResponse
>>> print repr(HttpRequest())
<HttpRequest
GET:{},
POST:{},
COOKIES:{},
META:{}>

>>> from django.core.handlers.wsgi import WSGIRequest
>>> print repr(WSGIRequest({'PATH_INFO': 'bogus', 'REQUEST_METHOD': 'bogus'}))
<WSGIRequest
GET:<QueryDict: {}>,
POST:<QueryDict: {}>,
COOKIES:{},
META:{...}>

>>> from django.core.handlers.modpython import ModPythonRequest
>>> class FakeModPythonRequest(ModPythonRequest):
...    def __init__(self, *args, **kwargs):
...        super(FakeModPythonRequest, self).__init__(*args, **kwargs)
...        self._get = self._post = self._meta = self._cookies = {}
>>> class Dummy:
...     def get_options(self):
...         return {}
>>> req = Dummy()
>>> req.uri = 'bogus'
>>> print repr(FakeModPythonRequest(req))
<ModPythonRequest
path:bogus,
GET:{},
POST:{},
COOKIES:{},
META:{}>

>>> from django.http import parse_cookie
>>> parse_cookie('invalid:key=true')
{}

>>> request = HttpRequest()
>>> print request.build_absolute_uri(location="https://www.example.com/asdf")
https://www.example.com/asdf
>>> request.get_host = lambda: 'www.example.com'
>>> request.path = ''
>>> print request.build_absolute_uri(location="/path/with:colons")
http://www.example.com/path/with:colons

# Test cookie datetime expiration logic
>>> from datetime import datetime, timedelta
>>> import time
>>> delta = timedelta(seconds=10)
>>> response = HttpResponse()
>>> expires = datetime.utcnow() + delta

# There is a timing weakness in this test; The
# expected result for max-age requires that there be
# a very slight difference between the evaluated expiration
# time, and the time evaluated in set_cookie(). If this
# difference doesn't exist, the cookie time will be
# 1 second larger. To avoid the problem, put in a quick sleep,
# which guarantees that there will be a time difference.
>>> time.sleep(0.001)
>>> response.set_cookie('datetime', expires=expires)
>>> datetime_cookie = response.cookies['datetime']
>>> datetime_cookie['max-age']
10
>>> response.set_cookie('datetime', expires=datetime(2028, 1, 1, 4, 5, 6))
>>> response.cookies['datetime']['expires']
'Sat, 01-Jan-2028 04:05:06 GMT'

# Test automatically setting cookie expires if only max_age is provided
>>> response.set_cookie('max_age', max_age=10)
>>> max_age_cookie = response.cookies['max_age']
>>> max_age_cookie['max-age']
10
>>> from django.utils.http import cookie_date
>>> import time
>>> max_age_cookie['expires'] == cookie_date(time.time()+10)
True
"""
<|endoftext|>