<commit_before>from mopidy import MopidyException

class MpdAckError(MopidyException):
    """
    Available MPD error codes::

        ACK_ERROR_NOT_LIST = 1
        ACK_ERROR_ARG = 2
        ACK_ERROR_PASSWORD = 3
        ACK_ERROR_PERMISSION = 4
        ACK_ERROR_UNKNOWN = 5
        ACK_ERROR_NO_EXIST = 50
        ACK_ERROR_PLAYLIST_MAX = 51
        ACK_ERROR_SYSTEM = 52
        ACK_ERROR_PLAYLIST_LOAD = 53
        ACK_ERROR_UPDATE_ALREADY = 54
        ACK_ERROR_PLAYER_SYNC = 55
        ACK_ERROR_EXIST = 56
    """

    def __init__(self, message=u'', error_code=0, index=0, command=u''):
        super(MpdAckError, self).__init__(message, error_code, index, command)
        self.message = message
        self.error_code = error_code
        self.index = index
        self.command = command

    def get_mpd_ack(self):
        """
        MPD error code format::

            ACK [%(error_code)i@%(index)i] {%(command)s} description
        """
        return u'ACK [%i@%i] {%s} %s' % (
            self.error_code, self.index, self.command, self.message)

class MpdArgError(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdArgError, self).__init__(*args, **kwargs)
        self.error_code = 2 # ACK_ERROR_ARG

class MpdPasswordError(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdPasswordError, self).__init__(*args, **kwargs)
        self.error_code = 3 # ACK_ERROR_PASSWORD

class MpdUnknownCommand(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdUnknownCommand, self).__init__(*args, **kwargs)
        self.message = u'unknown command "%s"' % self.command
        self.command = u''
        self.error_code = 5 # ACK_ERROR_UNKNOWN

class MpdNoExistError(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdNoExistError, self).__init__(*args, **kwargs)
        self.error_code = 50 # ACK_ERROR_NO_EXIST

class MpdNotImplemented(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdNotImplemented, self).__init__(*args, **kwargs)
        self.message = u'Not implemented'
<commit_msg>Expose MPD error codes as constants on the MpdAckError exception, to avoid use of magic values in the other exceptions<commit_after>from mopidy import MopidyException

class MpdAckError(MopidyException):
    """See fields on this class for available MPD error codes"""

    ACK_ERROR_NOT_LIST = 1
    ACK_ERROR_ARG = 2
    ACK_ERROR_PASSWORD = 3
    ACK_ERROR_PERMISSION = 4
    ACK_ERROR_UNKNOWN = 5
    ACK_ERROR_NO_EXIST = 50
    ACK_ERROR_PLAYLIST_MAX = 51
    ACK_ERROR_SYSTEM = 52
    ACK_ERROR_PLAYLIST_LOAD = 53
    ACK_ERROR_UPDATE_ALREADY = 54
    ACK_ERROR_PLAYER_SYNC = 55
    ACK_ERROR_EXIST = 56

    def __init__(self, message=u'', error_code=0, index=0, command=u''):
        super(MpdAckError, self).__init__(message, error_code, index, command)
        self.message = message
        self.error_code = error_code
        self.index = index
        self.command = command

    def get_mpd_ack(self):
        """
        MPD error code format::

            ACK [%(error_code)i@%(index)i] {%(command)s} description
        """
        return u'ACK [%i@%i] {%s} %s' % (
            self.error_code, self.index, self.command, self.message)

class MpdArgError(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdArgError, self).__init__(*args, **kwargs)
        self.error_code = MpdAckError.ACK_ERROR_ARG

class MpdPasswordError(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdPasswordError, self).__init__(*args, **kwargs)
        self.error_code = MpdAckError.ACK_ERROR_PASSWORD

class MpdUnknownCommand(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdUnknownCommand, self).__init__(*args, **kwargs)
        self.message = u'unknown command "%s"' % self.command
        self.command = u''
        self.error_code = MpdAckError.ACK_ERROR_UNKNOWN

class MpdNoExistError(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdNoExistError, self).__init__(*args, **kwargs)
        self.error_code = MpdAckError.ACK_ERROR_NO_EXIST

class MpdNotImplemented(MpdAckError):
    def __init__(self, *args, **kwargs):
        super(MpdNotImplemented, self).__init__(*args, **kwargs)
        self.message = u'Not implemented'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Views for CRAB services

.. versionadded:: 0.1.0
'''
from pyramid.view import view_config
from crabpy_pyramid.utils import range_return

@view_config(
    route_name='list_gewesten',
    renderer='crab_listjson', accept='application/json'
)
def list_gewesten(request):
    Gateway = request.crab_gateway()
    gewesten = Gateway.list_gewesten()
    total = len(gewesten)
    r = range_return(request, total)
    return gewesten[r[0]: r[1]]

@view_config(
    route_name='get_gewest_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_gewest_by_id(request):
    Gateway = request.crab_gateway()
    gewest_id = int(request.matchdict.get('gewest_id'))
    return Gateway.get_gewest_by_id(gewest_id)

@view_config(
    route_name='list_provincies',
    renderer='crab_listjson', accept='application/json'
)
def list_provincies(request):
    Gateway = request.crab_gateway()
    gewest_id =
    provincies = Gateway.list_provincies()

@view_config(
    route_name='list_gemeenten_crab',
    renderer='crab_listjson', accept='application/json'
)
def list_gemeenten_crab(request):
    Gateway = request.crab_gateway()
    sort = request.params.get('sort', 1)
    gewest_id = request.matchdict.get('gewest_id')
    gemeenten = Gateway.list_gemeenten(gewest_id, sort)
    total = len(gemeenten)
    r = range_return(request, total)
    return gemeenten[r[0]: r[1]]

@view_config(
    route_name='get_gemeente_crab',
    renderer='crab_itemjson', accept='application/json'
)
def get_gemeente_crab(request):
    Gateway = request.crab_gateway()
    gemeente_id = request.matchdict.get('gemeente_id')
    if len(gemeente_id) == 5:
        return Gateway.get_gemeente_by_niscode(gemeente_id)
    else:
        return Gateway.get_gemeente_by_id(gemeente_id)

@view_config(
    route_name='list_straten',
    renderer='crab_listjson', accept='application/json'
)
def list_straten(request):
    Gateway = request.crab_gateway()
    gemeente_id = request.matchdict.get('gemeente_id')
    if len(gemeente_id)==5:
        gemeente_id = Gateway.get_gemeente_by_niscode(gemeente_id)
    straten = Gateway.list_straten(gemeente_id)
    total = len(straten)
    r = range_return(request, total)
    return straten[r[0]: r[1]]

@view_config(
    route_name='get_straat_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_straat_by_id(request):
    Gateway = request.crab_gateway()
    straat_id = request.matchdict.get('straat_id')
    return Gateway.get_straat_by_id(straat_id)

@view_config(
    route_name='list_huisnummers',
    renderer='crab_listjson', accept='application/json'
)
def list_huisnummers(request):
    Gateway = request.crab_gateway()
    straat_id = request.matchdict.get('straat_id')
    huisnummers = Gateway.list_huisnummers_by_straat(straat_id)
    total = len(huisnummers)
    r = range_return(request, total)
    return huisnummers[r[0]: r[1]]

@view_config(
    route_name='get_huisnummer_by_straat_and_label',
    renderer='crab_itemjson', accept='application/json'
)
def get_huisnummer_by_straat_and_label(request):
    Gateway = request.crab_gateway()
    straat_id = request.matchdict.get('straat_id')
    huisnummer = request.matchdict.get('huisnummer_label')
    return Gateway.get_huisnummer_by_nummer_and_straat(huisnummer, straat_id)

@view_config(
    route_name='get_huisnummer_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_huisnummer_by_id(request):
    Gateway = request.crab_gateway()
    huisnummer_id = request.matchdict.get('huisnummer_id')
    return Gateway.get_huisnummer_by_id(huisnummer_id)

@view_config(
    route_name='list_percelen',
    renderer='crab_listjson', accept='application/json'
)
def list_percelen(request):
    Gateway = request.crab_gateway()
    huisnummer_id = request.matchdict.get('huisnummer_id')
    percelen = Gateway.list_percelen_by_huisnummer(huisnummer_id)
    total = len(percelen)
    r = range_return(request, total)
    return percelen[r[0]: r[1]]

@view_config(
    route_name='get_perceel_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_perceel_by_id(request):
    Gateway = request.crab_gateway()
    perceel_id = request.matchdict.get('perceel_id1')+'/'+request.matchdict.get('perceel_id2')
    return Gateway.get_perceel_by_id(perceel_id)

@view_config(
    route_name='list_gebouwen',
    renderer='crab_listjson', accept='application/json'
)
def list_gebouwen(request):
    Gateway = request.crab_gateway()
    huisnummer_id = request.matchdict.get('huisnummer_id')
    gebouwen = Gateway.list_gebouwen_by_huisnummer(huisnummer_id)
    total = len(gebouwen)
    r = range_return(request, total)
    return gebouwen[r[0]: r[1]]

@view_config(
    route_name='get_gebouw_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_gebouw_by_id(request):
    Gateway = request.crab_gateway()
    gebouw_id = request.matchdict.get('gebouw_id')
    return Gateway.get_gebouw_by_id(gebouw_id)

@view_config(
    route_name='get_wegobject',
    renderer='crab_itemjson', accept='application/json'
)
def get_wegobject(request):
    Gateway = request.crab_gateway()
    wegobject_id = request.matchdict.get('wegobject_id')
    return Gateway.get_wegobject_by_id(wegobject_id)
<commit_msg>added view list_provincies refs #13<commit_after># -*- coding: utf-8 -*-
'''
Views for CRAB services

.. versionadded:: 0.1.0
'''
from pyramid.view import view_config
from crabpy_pyramid.utils import range_return

@view_config(
    route_name='list_gewesten',
    renderer='crab_listjson', accept='application/json'
)
def list_gewesten(request):
    Gateway = request.crab_gateway()
    gewesten = Gateway.list_gewesten()
    total = len(gewesten)
    r = range_return(request, total)
    return gewesten[r[0]: r[1]]

@view_config(
    route_name='get_gewest_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_gewest_by_id(request):
    Gateway = request.crab_gateway()
    gewest_id = int(request.matchdict.get('gewest_id'))
    return Gateway.get_gewest_by_id(gewest_id)

@view_config(
    route_name='list_provincies',
    renderer='crab_listjson', accept='application/json'
)
def list_provincies(request):
    Gateway = request.crab_gateway()
    gewest_id = int(request.matchdict.get('gewest_id'))
    provincies = Gateway.list_provincies(gewest_id)
    return provincies

@view_config(
    route_name='list_gemeenten_crab',
    renderer='crab_listjson', accept='application/json'
)
def list_gemeenten_crab(request):
    Gateway = request.crab_gateway()
    sort = request.params.get('sort', 1)
    gewest_id = request.matchdict.get('gewest_id')
    gemeenten = Gateway.list_gemeenten(gewest_id, sort)
    total = len(gemeenten)
    r = range_return(request, total)
    return gemeenten[r[0]: r[1]]

@view_config(
    route_name='get_gemeente_crab',
    renderer='crab_itemjson', accept='application/json'
)
def get_gemeente_crab(request):
    Gateway = request.crab_gateway()
    gemeente_id = request.matchdict.get('gemeente_id')
    if len(gemeente_id) == 5:
        return Gateway.get_gemeente_by_niscode(gemeente_id)
    else:
        return Gateway.get_gemeente_by_id(gemeente_id)

@view_config(
    route_name='list_straten',
    renderer='crab_listjson', accept='application/json'
)
def list_straten(request):
    Gateway = request.crab_gateway()
    gemeente_id = request.matchdict.get('gemeente_id')
    if len(gemeente_id)==5:
        gemeente_id = Gateway.get_gemeente_by_niscode(gemeente_id)
    straten = Gateway.list_straten(gemeente_id)
    total = len(straten)
    r = range_return(request, total)
    return straten[r[0]: r[1]]

@view_config(
    route_name='get_straat_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_straat_by_id(request):
    Gateway = request.crab_gateway()
    straat_id = request.matchdict.get('straat_id')
    return Gateway.get_straat_by_id(straat_id)

@view_config(
    route_name='list_huisnummers',
    renderer='crab_listjson', accept='application/json'
)
def list_huisnummers(request):
    Gateway = request.crab_gateway()
    straat_id = request.matchdict.get('straat_id')
    huisnummers = Gateway.list_huisnummers_by_straat(straat_id)
    total = len(huisnummers)
    r = range_return(request, total)
    return huisnummers[r[0]: r[1]]

@view_config(
    route_name='get_huisnummer_by_straat_and_label',
    renderer='crab_itemjson', accept='application/json'
)
def get_huisnummer_by_straat_and_label(request):
    Gateway = request.crab_gateway()
    straat_id = request.matchdict.get('straat_id')
    huisnummer = request.matchdict.get('huisnummer_label')
    return Gateway.get_huisnummer_by_nummer_and_straat(huisnummer, straat_id)

@view_config(
    route_name='get_huisnummer_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_huisnummer_by_id(request):
    Gateway = request.crab_gateway()
    huisnummer_id = request.matchdict.get('huisnummer_id')
    return Gateway.get_huisnummer_by_id(huisnummer_id)

@view_config(
    route_name='list_percelen',
    renderer='crab_listjson', accept='application/json'
)
def list_percelen(request):
    Gateway = request.crab_gateway()
    huisnummer_id = request.matchdict.get('huisnummer_id')
    percelen = Gateway.list_percelen_by_huisnummer(huisnummer_id)
    total = len(percelen)
    r = range_return(request, total)
    return percelen[r[0]: r[1]]

@view_config(
    route_name='get_perceel_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_perceel_by_id(request):
    Gateway = request.crab_gateway()
    perceel_id = request.matchdict.get('perceel_id1')+'/'+request.matchdict.get('perceel_id2')
    return Gateway.get_perceel_by_id(perceel_id)

@view_config(
    route_name='list_gebouwen',
    renderer='crab_listjson', accept='application/json'
)
def list_gebouwen(request):
    Gateway = request.crab_gateway()
    huisnummer_id = request.matchdict.get('huisnummer_id')
    gebouwen = Gateway.list_gebouwen_by_huisnummer(huisnummer_id)
    total = len(gebouwen)
    r = range_return(request, total)
    return gebouwen[r[0]: r[1]]

@view_config(
    route_name='get_gebouw_by_id',
    renderer='crab_itemjson', accept='application/json'
)
def get_gebouw_by_id(request):
    Gateway = request.crab_gateway()
    gebouw_id = request.matchdict.get('gebouw_id')
    return Gateway.get_gebouw_by_id(gebouw_id)

@view_config(
    route_name='get_wegobject',
    renderer='crab_itemjson', accept='application/json'
)
def get_wegobject(request):
    Gateway = request.crab_gateway()
    wegobject_id = request.matchdict.get('wegobject_id')
    return Gateway.get_wegobject_by_id(wegobject_id)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy as np
import inspect
import trimesh

from trimesh.exchange.binvox import voxelize_mesh
from trimesh import voxel as v

dir_current = os.path.dirname(
    os.path.abspath(
        inspect.getfile(
            inspect.currentframe())))
# the absolute path for our reference models
dir_models = os.path.abspath(
    os.path.join(dir_current, '..', 'models'))

def show(chair_mesh, chair_voxels, colors=(1, 1, 1, 0.3)):
    scene = chair_mesh.scene()
    scene.add_geometry(chair_voxels.as_boxes(colors=colors))
    scene.show()

if __name__ == '__main__':
    base_name = 'chair_model'
    chair_mesh = trimesh.load(os.path.join(dir_models, '%s.obj' % base_name))
    if isinstance(chair_mesh, trimesh.scene.Scene):
        chair_mesh = trimesh.util.concatenate([
            trimesh.Trimesh(mesh.vertices, mesh.faces)
            for mesh in chair_mesh.geometry.values()])

    binvox_path = os.path.join(dir_models, '%s.binvox' % base_name)
    chair_voxels = trimesh.load(binvox_path)

    chair_voxels = v.VoxelGrid(chair_voxels.encoding.dense, chair_voxels.transform)

    print('white: voxelized chair (binvox, exact)')
    show(chair_mesh, voxelize_mesh(chair_mesh, exact=True), colors=(1, 1, 1, 0.3))

    print('red: binvox-loaded chair')
    show(chair_mesh, chair_voxels, colors=(1, 0, 0, 0.3))

    voxelized_chair_mesh = chair_mesh.voxelized(np.max(chair_mesh.extents) / 32)
    print('green: voxelized chair (default).')
    show(chair_mesh, voxelized_chair_mesh, colors=(0, 1, 0, 0.3))

    shape = (50, 17, 63)
    revox = chair_voxels.revoxelized(shape)
    print('cyan: revoxelized.')
    show(chair_mesh, revox, colors=(0, 1, 1, 0.3))

    values = chair_voxels.encoding.dense.copy()
    values[:values.shape[0] // 2] = 0
    stripped = v.VoxelGrid(values, chair_voxels.transform.copy()).strip()
    print('yellow: stripped halved voxel grid. Transform is updated appropriately')
    show(chair_mesh, stripped, colors=(1, 1, 0, 0.3))

    transform = np.eye(4)
    transform[:3] += np.random.normal(size=(3, 4)) * 0.2
    transformed_chair_mesh = chair_mesh.copy().apply_transform(transform)
    print('original transform volume: %s'
          % str(chair_voxels.element_volume))

    chair_voxels.apply_transform(transform)
    print('warped transform volume:   %s' % str(chair_voxels.element_volume))
    print('blue: transformed voxels. Transformation is lazy, and each voxel is '
          'no longer a cube.')
    show(transformed_chair_mesh, chair_voxels, colors=(0, 0, 1, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='subdivide').fill()
    print('green: subdivided')
    show(chair_mesh, voxelized, colors=(0, 1, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='ray')
    print('red: ray. Poor performance on thin structures')
    show(chair_mesh, voxelized, colors=(1, 0, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='binvox')
    print('red: binvox (default). Poor performance on thin structures')
    show(chair_mesh, voxelized, colors=(1, 0, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='binvox', wireframe=True)
    print('green: binvox (wireframe). Still doesn\'t capture all thin structures')
    show(chair_mesh, voxelized, colors=(0, 1, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='binvox', exact=True)
    print('blue: binvox (exact). Does a good job')
    show(chair_mesh, voxelized, colors=(0, 0, 1, 0.3))

    voxelized = chair_mesh.voxelized(
        pitch=0.02,
        method='binvox',
        exact=True,
        downsample_factor=2,
        downsample_threshold=1)
    print('red: binvox (exact downsampled) surface')
    show(chair_mesh, voxelized, colors=(1, 0, 0, 0.3))

    chair_voxels = chair_mesh.voxelized(pitch=0.02, method='binvox', exact=True)

    voxelized = chair_voxels.copy().fill(method='base')
    print('blue: binvox (exact) filled (base). Gets a bit overly excited')
    show(chair_mesh, voxelized, colors=(0, 0, 1, 0.3))

    voxelized = chair_voxels.copy().fill(method='orthographic')
    print('green: binvox (exact) filled (orthographic). '
          'Doesn\'t do much as should be expected')
    show(chair_mesh, voxelized, colors=(0, 1, 0, 0.3))
<commit_msg>flake8<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy as np
import inspect
import trimesh

from trimesh.exchange.binvox import voxelize_mesh
from trimesh import voxel as v

dir_current = os.path.dirname(
    os.path.abspath(
        inspect.getfile(
            inspect.currentframe())))
# the absolute path for our reference models
dir_models = os.path.abspath(
    os.path.join(dir_current, '..', 'models'))

def show(chair_mesh, chair_voxels, colors=(1, 1, 1, 0.3)):
    scene = chair_mesh.scene()
    scene.add_geometry(chair_voxels.as_boxes(colors=colors))
    scene.show()

if __name__ == '__main__':
    base_name = 'chair_model'
    chair_mesh = trimesh.load(os.path.join(dir_models, '%s.obj' % base_name))
    if isinstance(chair_mesh, trimesh.scene.Scene):
        chair_mesh = trimesh.util.concatenate([
            trimesh.Trimesh(mesh.vertices, mesh.faces)
            for mesh in chair_mesh.geometry.values()])

    binvox_path = os.path.join(dir_models, '%s.binvox' % base_name)
    chair_voxels = trimesh.load(binvox_path)

    chair_voxels = v.VoxelGrid(chair_voxels.encoding.dense, chair_voxels.transform)

    print('white: voxelized chair (binvox, exact)')
    show(chair_mesh, voxelize_mesh(chair_mesh, exact=True), colors=(1, 1, 1, 0.3))

    print('red: binvox-loaded chair')
    show(chair_mesh, chair_voxels, colors=(1, 0, 0, 0.3))

    voxelized_chair_mesh = chair_mesh.voxelized(np.max(chair_mesh.extents) / 32)
    print('green: voxelized chair (default).')
    show(chair_mesh, voxelized_chair_mesh, colors=(0, 1, 0, 0.3))

    shape = (50, 17, 63)
    revox = chair_voxels.revoxelized(shape)
    print('cyan: revoxelized.')
    show(chair_mesh, revox, colors=(0, 1, 1, 0.3))

    values = chair_voxels.encoding.dense.copy()
    values[:values.shape[0] // 2] = 0
    stripped = v.VoxelGrid(values, chair_voxels.transform.copy()).strip()
    print('yellow: stripped halved voxel grid. Transform is updated appropriately')
    show(chair_mesh, stripped, colors=(1, 1, 0, 0.3))

    transform = np.eye(4)
    transform[:3] += np.random.normal(size=(3, 4)) * 0.2
    transformed_chair_mesh = chair_mesh.copy().apply_transform(transform)
    print('original transform volume: %s'
          % str(chair_voxels.element_volume))

    chair_voxels.apply_transform(transform)
    print('warped transform volume:   %s' % str(chair_voxels.element_volume))
    print('blue: transformed voxels')
    print('Transformation is lazy, and each voxel is no longer a cube.')
    show(transformed_chair_mesh, chair_voxels, colors=(0, 0, 1, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='subdivide').fill()
    print('green: subdivided')
    show(chair_mesh, voxelized, colors=(0, 1, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='ray')
    print('red: ray. Poor performance on thin structures')
    show(chair_mesh, voxelized, colors=(1, 0, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='binvox')
    print('red: binvox (default). Poor performance on thin structures')
    show(chair_mesh, voxelized, colors=(1, 0, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='binvox', wireframe=True)
    print('green: binvox (wireframe). Still doesn\'t capture all thin structures')
    show(chair_mesh, voxelized, colors=(0, 1, 0, 0.3))

    voxelized = chair_mesh.voxelized(pitch=0.02, method='binvox', exact=True)
    print('blue: binvox (exact). Does a good job')
    show(chair_mesh, voxelized, colors=(0, 0, 1, 0.3))

    voxelized = chair_mesh.voxelized(
        pitch=0.02,
        method='binvox',
        exact=True,
        downsample_factor=2,
        downsample_threshold=1)
    print('red: binvox (exact downsampled) surface')
    show(chair_mesh, voxelized, colors=(1, 0, 0, 0.3))

    chair_voxels = chair_mesh.voxelized(pitch=0.02, method='binvox', exact=True)

    voxelized = chair_voxels.copy().fill(method='base')
    print('blue: binvox (exact) filled (base). Gets a bit overly excited')
    show(chair_mesh, voxelized, colors=(0, 0, 1, 0.3))

    voxelized = chair_voxels.copy().fill(method='orthographic')
    print('green: binvox (exact) filled (orthographic).')
    print("Doesn't do much as should be expected")
    show(chair_mesh, voxelized, colors=(0, 1, 0, 0.3))
<|endoftext|>


<commit_before>"""
print the environ dict
"""

from zunzuncito import tools

class APIResource(object):

    def dispatch(self, request, response):

        request.log.debug(tools.log_json({
            'API': request.version,
            'Method': request.method,
            'URI': request.URI,
            'vroot': request.vroot
        }, True))

        return request.environ.get('HTTP_USER_AGENT').encode('utf-8')
<commit_msg>	modified:   zun_ua.py<commit_after>"""
print the environ dict
"""

from zunzuncito import tools

class APIResource(object):

    def dispatch(self, request, response):

        request.log.debug(tools.log_json({
            'API': request.version,
            'Method': request.method,
            'URI': request.URI,
            'vroot': request.vroot
        }, True))

        b = request.environ.get('HTTP_USER_AGENT')

        for k, v in request.environ.items():
            # convert all string values to unicode values and replace
            # malformed data with a suitable
            # replacement marker.
            if isinstance(v, str):
                request.environ[k] = v.decode('utf-8', 'replace')

        a = request.environ.get('HTTP_USER_AGENT')
        return str(type(b)) + str(type(a))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os,sys,getopt,tarfile
import getpass

from distutils.spawn import find_executable

import time
import socket
import optparse
import subprocess
import multiprocessing

from swiftclient import Connection

from swiftclient import shell
from swiftclient import RequestException
from swiftclient.exceptions import ClientException
from swiftclient.multithreading import OutputManager

try:
    from scandir import walk
except:
    print('importing os.walk instead of scandir.walk')
    from os import walk

swift_auth=os.environ.get("ST_AUTH")
swift_auth_token=os.environ.get("OS_AUTH_TOKEN")
storage_url=os.environ.get("OS_STORAGE_URL")
haz_pigz=False

# define minimum parser object to allow swiftstack shell to run
def shell_minimal_options():
   global swift_auth,swift_auth_token,storage_url

   parser = optparse.OptionParser()

   parser.add_option('-A', '--auth', dest='auth',
      default=swift_auth)
   parser.add_option('-V', '--auth-version',
      default=os.environ.get('ST_AUTH_VERSION',
         (os.environ.get('OS_AUTH_VERSION','1.0'))))
   parser.add_option('-U', '--user', dest='user',
      default=os.environ.get('ST_USER'))
   parser.add_option('-K', '--key', dest='key',
      default=os.environ.get('ST_KEY'))

   parser.add_option('--os_auth_token',default=swift_auth_token)
   parser.add_option('--os_storage_url',default=storage_url)

   parser.add_option('--os_username')
   parser.add_option('--os_password')
   parser.add_option('--os_auth_url')

   parser.add_option('--os_user_id')
   parser.add_option('--os_user_domain_id')
   parser.add_option('--os_user_domain_name')
   parser.add_option('--os_tenant_id')
   parser.add_option('--os_tenant_name')
   parser.add_option('--os_project_id')
   parser.add_option('--os_project_domain_id')
   parser.add_option('--os_project_name')
   parser.add_option('--os_project_domain_name')
   parser.add_option('--os_service_type')
   parser.add_option('--os_endpoint_type')
   parser.add_option('--os_region_name')

   parser.add_option('-v', '--verbose', action='count', dest='verbose',
       default=1, help='Print more info.')

   return parser

# wrapper function for swiftstack shell functions
def sw_shell(sw_fun,*args):
   global swift_auth_token,storage_url

   if swift_auth_token and storage_url:
      args=args+("--os_auth_token",swift_auth_token,
         "--os_storage_url",storage_url)

   args = ('',) + args
   with OutputManager() as output:
      parser = shell_minimal_options()
      try:
         sw_fun(parser, list(args), output)
      except (ClientException, RequestException, socket.error) as err:
         output.error(str(err))

def sw_download(*args):
   sw_shell(shell.st_download,*args)

def sw_upload(*args):
   sw_shell(shell.st_upload,*args)

def sw_post(*args):
   sw_shell(shell.st_post,*args)

# suffix of archive files
tar_suffix=".tar.gz"
root_id=".root"

# True if 1st char of path member is '.' else False
def is_hidden_dir(dir_name):
   for item in dir_name.split('/'):
      if item[0]=='.':
         return True

   return False

def print_flush(str):
   sys.stdout.write(str+'\n')
   sys.stdout.flush()

# apparently getpid is ok because it's different between mp tasks
def unique_id():
   return str(os.getpid())

def create_tar_file(filename,src_path,file_list):
   global haz_pigz

   # only archive src_path directory
   tar_params=["tar","cvf",filename,"--directory="+src_path,"--no-recursion"]
   if haz_pigz:
      tar_params=tar_params+["--use-compress-program=pigz"]

   # include directory itself in archive for ownership & permissions
   tar_params=tar_params+['.']

   # generate external file list only if files to be archived
   if file_list:
      tmp_file="/tmp/.tar."+unique_id()
      with open(tmp_file,"w") as f:
         for file in file_list:
            f.write("-- \""+file+"\"\n")
      tar_params=tar_params+["-T",tmp_file]

   ret=subprocess.call(tar_params)
   if ret>0:
      sys.stderr.write('***** TAR ERROR %s, command: %s *****\n' %
         (ret,tar_params))

   if file_list:
      os.unlink(tmp_file)

def upload_file_to_swift(filename,swiftname,container):
   sw_upload("--object-name="+swiftname,
      "--segment-size=2147483648",
      "--use-slo",
      "--segment-container=.segments_"+container,
      "--header=X-Object-Meta-Uploaded-by:"+getpass.getuser(),
      container,filename)

def archive_tar_file(src_path,file_list,container,tmp_dir,pre_path):
   global tar_suffix

   # archive_name is name for archived object
   archive_name=pre_path+tar_suffix
   # temp_archive_name is name of local tar file
   temp_archive_name=unique_id()+os.path.basename(archive_name)
   if tmp_dir:
      temp_archive_name=os.path.join(tmp_dir,temp_archive_name)

   # Create local tar file
   create_tar_file(temp_archive_name,src_path,file_list)

   # Upload tar file to container as 'archive_name'
   upload_file_to_swift(temp_archive_name,archive_name,container)

   # Delete local tar file
   os.unlink(temp_archive_name)

def is_child_or_sib(dir_name,last_dir):
   dname=os.path.dirname(dir_name)
   return (dname==last_dir or dname==os.path.dirname(last_dir))

# param order: [src_path,file_list,container,tmp_dir,pre_path]
def archive_worker(item):
   archive_tar_file(item[0],item[1],item[2],item[3],item[4])

def archive_to_swift(local_dir,container,no_hidden,tmp_dir,prefix,par,subtree,
   meta):
   last_dir=""
   archive=[]

   # combine 1st param and meta list into individual parameters
   param=[container]+meta
   sw_post(*param)

   for dir_name, subdir_list, file_list in mywalk(local_dir):
      rel_path=os.path.relpath(dir_name,local_dir)
      if (not (no_hidden and is_hidden_dir(rel_path))):
         # if files in root directory use basename of root
         if rel_path==".":
            rel_path=os.path.basename(dir_name)+root_id

         if (not subtree) or (is_subtree(subtree,dir_name)):
            archive.append([dir_name,file_list,container,tmp_dir,
               os.path.join(prefix,rel_path)])

         last_dir=dir_name

   archive_pool=multiprocessing.Pool(par)
   archive_pool.map(archive_worker,archive)

# parse name into directory tree
def create_local_path(local_dir,archive_name):
   global tar_suffix

   path=os.path.join(local_dir,archive_name)
   if path.endswith(tar_suffix):
      path=path[:-len(tar_suffix)]

   if not os.path.exists(path):
      os.makedirs(path)

   return path

def create_sw_conn():
   global swift_auth,swift_auth_token,storage_url

   if swift_auth_token and storage_url:
      return Connection(preauthtoken=swift_auth_token,preauthurl=storage_url)

   if swift_auth:
      swift_user=os.environ.get("ST_USER")
      swift_key=os.environ.get("ST_KEY")

      if swift_user and swift_key:
         return Connection(authurl=swift_auth,user=swift_user,key=swift_key)

   print("Error: Swift environment not configured!")
   sys.exit()

def extract_tar_file(tarfile,termpath):
   global haz_pigz

   tar_params=["tar","xvf",tarfile,"--directory="+termpath]
   if haz_pigz:
      tar_params=tar_params+["--use-compress-program=pigz"]

   ret=subprocess.call(tar_params)
   if ret > 0:
      sys.stderr.write('***** TAR ERROR %s, command: %s *****\n' %
         (ret,tar_params))

def retrieve_tar_file(tmp_dir,container,obj_name,local_dir,prefix):
   global tar_suffix
   global root_id

   # download tar file and extract into terminal directory
   temp_file=unique_id()+tar_suffix
   if tmp_dir:
      temp_file=os.path.join(tmp_dir,temp_file)

   sw_download("--output="+temp_file,container,obj_name)

   # strip prefix and if next char is /, strip it too
   if prefix and obj_name.startswith(prefix):
      obj_name=obj_name[len(prefix):]
      if obj_name[0]=='/':
         obj_name=obj_name[1:]

   # if bundle, extract using tar embedded paths
   if obj_name.endswith(root_id+tar_suffix):
      term_path=local_dir
   else:
      term_path=create_local_path(local_dir,obj_name)

   extract_tar_file(temp_file,term_path)

   os.unlink(temp_file)

def extract_worker(item):
   retrieve_tar_file(item[0],item[1],item[2],item[3],item[4])

def extract_to_local(local_dir,container,no_hidden,tmp_dir,prefix,par):
   global tar_suffix
   global root_id

   swift_conn=create_sw_conn()
   if swift_conn:
      extract=[]

      try:
         headers,objs=swift_conn.get_container(container,prefix=prefix,
            full_listing=True)
         for obj in objs:
            if obj['name'].endswith(tar_suffix):
               if no_hidden and is_hidden_dir(obj['name']):
                  continue

               # param order: [tmp_dir,container,obj_name,local_dir,prefix]
               extract.append([tmp_dir,container,obj['name'],local_dir,prefix])
      except ClientException:
         print("Error: cannot access Swift container '%s'!" % container)

      extract_pool=multiprocessing.Pool(par)
      extract_pool.map(extract_worker,extract)

      swift_conn.close()

def usage():
   print("archive [parameters]")
   print("Parameters:")
   print("\t-l local_directory (default .)")
   print("\t-c container (required)")
   print("\t-x (extract from container to local directory)")
   print("\t-n (no hidden directories)")
   print("\t-t temp_dir (directory for temp files)")
   print("\t-a auth_token (default OS_AUTH_TOKEN)")
   print("\t-s storage_url (default OS_STORAGE_URL)")
   print("\t-p prefix")
   print("\t-P parallel_instances (default 3)")
   print("\t-m name:value (set container metadata)")

# is path a child of tree?
def is_subtree(tree,path):
   tree_sp=tree.split('/')
   path_sp=path.split('/')

   if len(path_sp)<len(tree_sp):
      return False

   for t,p in zip(tree_sp,path_sp):
      if t!=p:
         return False

   return True

def validate_dir(path,param):
   if not os.path.isdir(path):
      print("Error: %s '%s' is not accessible!" % (param,path))
      sys.exit()

   if path[-1]=='/':
      path=path[:-1]

   return(path)

def mywalk(top, skipdirs=['.snapshot',]):
    """ returns subset of os.walk  """
    for root, dirs, files in walk(top,topdown=True,onerror=walkerr):
        for skipdir in skipdirs:
            if skipdir in dirs:
                dirs.remove(skipdir)  # don't visit this directory
        yield root, dirs, files

def walkerr(oserr):
    sys.stderr.write(str(oserr))
    sys.stderr.write('\n')
    return 0

def main(argv):
   global swift_auth_token
   global storage_url
   global haz_pigz

   meta=[]
   sub_tree=""
   local_dir="."
   container=""
   tmp_dir=""
   extract=False
   no_hidden=False
   prefix=""
   par=3

   try:
      opts,args=getopt.getopt(argv,"l:c:t:a:s:p:P:S:m:xnh")
   except getopt.GetoptError:
      usage()
      sys.exit()

   for opt,arg in opts:
      if opt in ("-h"):
         usage()
         sys.exit()
      elif opt in ("-l"): # override default local directory
         local_dir=validate_dir(arg,"local")
      elif opt in ("-c"): # set container
         container=arg
      elif opt in ("-t"): # temp file directory
         tmp_dir=validate_dir(arg,"tmp_dir")
      elif opt in ("-a"): # override auth_token
         swift_auth_token=arg
      elif opt in ("-s"): # override storage URL
         storage_url=arg
      elif opt in ("-p"): # set prefix
         prefix=arg
      elif opt in ("-P"): # set parallel threads
         par=int(arg)
      elif opt in ("-x"): # extract mode
         extract=True
      elif opt in ("-n"): # set no-hidden flag to skip .*
         no_hidden=True
      elif opt in ("-S"): # specify optional sub-tree
         sub_tree=validate_dir(os.path.join(local_dir,arg),"subtree")
      elif opt in ("-m"): # specify container metadata
         if arg.count(':')==1:
            meta.append("-m"+arg)
         else:
            print("Error: metadata not in format key:value!")
            sys.exit()

   if not container:
      usage()
   else:
      if find_executable("pigz"):
         haz_pigz=True

      if extract:
         extract_to_local(local_dir,container,no_hidden,tmp_dir,prefix,par)
      else:
         archive_to_swift(local_dir,container,no_hidden,tmp_dir,prefix,par,
            sub_tree,meta)

if __name__=="__main__":
   main(sys.argv[1:])
<commit_msg>cleaner way of calling sw_post<commit_after>#!/usr/bin/env python3

import os,sys,getopt,tarfile
import getpass

from distutils.spawn import find_executable

import time
import socket
import optparse
import subprocess
import multiprocessing

from swiftclient import Connection

from swiftclient import shell
from swiftclient import RequestException
from swiftclient.exceptions import ClientException
from swiftclient.multithreading import OutputManager

try:
    from scandir import walk
except:
    print('importing os.walk instead of scandir.walk')
    from os import walk

swift_auth=os.environ.get("ST_AUTH")
swift_auth_token=os.environ.get("OS_AUTH_TOKEN")
storage_url=os.environ.get("OS_STORAGE_URL")
haz_pigz=False

# define minimum parser object to allow swiftstack shell to run
def shell_minimal_options():
   global swift_auth,swift_auth_token,storage_url

   parser = optparse.OptionParser()

   parser.add_option('-A', '--auth', dest='auth',
      default=swift_auth)
   parser.add_option('-V', '--auth-version',
      default=os.environ.get('ST_AUTH_VERSION',
         (os.environ.get('OS_AUTH_VERSION','1.0'))))
   parser.add_option('-U', '--user', dest='user',
      default=os.environ.get('ST_USER'))
   parser.add_option('-K', '--key', dest='key',
      default=os.environ.get('ST_KEY'))

   parser.add_option('--os_auth_token',default=swift_auth_token)
   parser.add_option('--os_storage_url',default=storage_url)

   parser.add_option('--os_username')
   parser.add_option('--os_password')
   parser.add_option('--os_auth_url')

   parser.add_option('--os_user_id')
   parser.add_option('--os_user_domain_id')
   parser.add_option('--os_user_domain_name')
   parser.add_option('--os_tenant_id')
   parser.add_option('--os_tenant_name')
   parser.add_option('--os_project_id')
   parser.add_option('--os_project_domain_id')
   parser.add_option('--os_project_name')
   parser.add_option('--os_project_domain_name')
   parser.add_option('--os_service_type')
   parser.add_option('--os_endpoint_type')
   parser.add_option('--os_region_name')

   parser.add_option('-v', '--verbose', action='count', dest='verbose',
       default=1, help='Print more info.')

   return parser

# wrapper function for swiftstack shell functions
def sw_shell(sw_fun,*args):
   global swift_auth_token,storage_url

   if swift_auth_token and storage_url:
      args=args+("--os_auth_token",swift_auth_token,
         "--os_storage_url",storage_url)

   args = ('',) + args
   with OutputManager() as output:
      parser = shell_minimal_options()
      try:
         sw_fun(parser, list(args), output)
      except (ClientException, RequestException, socket.error) as err:
         output.error(str(err))

def sw_download(*args):
   sw_shell(shell.st_download,*args)

def sw_upload(*args):
   sw_shell(shell.st_upload,*args)

def sw_post(*args):
   sw_shell(shell.st_post,*args)

# suffix of archive files
tar_suffix=".tar.gz"
root_id=".root"

# True if 1st char of path member is '.' else False
def is_hidden_dir(dir_name):
   for item in dir_name.split('/'):
      if item[0]=='.':
         return True

   return False

def print_flush(str):
   sys.stdout.write(str+'\n')
   sys.stdout.flush()

# apparently getpid is ok because it's different between mp tasks
def unique_id():
   return str(os.getpid())

def create_tar_file(filename,src_path,file_list):
   global haz_pigz

   # only archive src_path directory
   tar_params=["tar","cvf",filename,"--directory="+src_path,"--no-recursion"]
   if haz_pigz:
      tar_params=tar_params+["--use-compress-program=pigz"]

   # include directory itself in archive for ownership & permissions
   tar_params=tar_params+['.']

   # generate external file list only if files to be archived
   if file_list:
      tmp_file="/tmp/.tar."+unique_id()
      with open(tmp_file,"w") as f:
         for file in file_list:
            f.write("-- \""+file+"\"\n")
      tar_params=tar_params+["-T",tmp_file]

   ret=subprocess.call(tar_params)
   if ret>0:
      sys.stderr.write('***** TAR ERROR %s, command: %s *****\n' %
         (ret,tar_params))

   if file_list:
      os.unlink(tmp_file)

def upload_file_to_swift(filename,swiftname,container):
   sw_upload("--object-name="+swiftname,
      "--segment-size=2147483648",
      "--use-slo",
      "--segment-container=.segments_"+container,
      "--header=X-Object-Meta-Uploaded-by:"+getpass.getuser(),
      container,filename)

def archive_tar_file(src_path,file_list,container,tmp_dir,pre_path):
   global tar_suffix

   # archive_name is name for archived object
   archive_name=pre_path+tar_suffix
   # temp_archive_name is name of local tar file
   temp_archive_name=unique_id()+os.path.basename(archive_name)
   if tmp_dir:
      temp_archive_name=os.path.join(tmp_dir,temp_archive_name)

   # Create local tar file
   create_tar_file(temp_archive_name,src_path,file_list)

   # Upload tar file to container as 'archive_name'
   upload_file_to_swift(temp_archive_name,archive_name,container)

   # Delete local tar file
   os.unlink(temp_archive_name)

def is_child_or_sib(dir_name,last_dir):
   dname=os.path.dirname(dir_name)
   return (dname==last_dir or dname==os.path.dirname(last_dir))

# param order: [src_path,file_list,container,tmp_dir,pre_path]
def archive_worker(item):
   archive_tar_file(item[0],item[1],item[2],item[3],item[4])

def archive_to_swift(local_dir,container,no_hidden,tmp_dir,prefix,par,subtree,
   meta):
   last_dir=""
   archive=[]

   sw_post(container,*meta)

   for dir_name, subdir_list, file_list in mywalk(local_dir):
      rel_path=os.path.relpath(dir_name,local_dir)
      if (not (no_hidden and is_hidden_dir(rel_path))):
         # if files in root directory use basename of root
         if rel_path==".":
            rel_path=os.path.basename(dir_name)+root_id

         if (not subtree) or (is_subtree(subtree,dir_name)):
            archive.append([dir_name,file_list,container,tmp_dir,
               os.path.join(prefix,rel_path)])

         last_dir=dir_name

   archive_pool=multiprocessing.Pool(par)
   archive_pool.map(archive_worker,archive)

# parse name into directory tree
def create_local_path(local_dir,archive_name):
   global tar_suffix

   path=os.path.join(local_dir,archive_name)
   if path.endswith(tar_suffix):
      path=path[:-len(tar_suffix)]

   if not os.path.exists(path):
      os.makedirs(path)

   return path

def create_sw_conn():
   global swift_auth,swift_auth_token,storage_url

   if swift_auth_token and storage_url:
      return Connection(preauthtoken=swift_auth_token,preauthurl=storage_url)

   if swift_auth:
      swift_user=os.environ.get("ST_USER")
      swift_key=os.environ.get("ST_KEY")

      if swift_user and swift_key:
         return Connection(authurl=swift_auth,user=swift_user,key=swift_key)

   print("Error: Swift environment not configured!")
   sys.exit()

def extract_tar_file(tarfile,termpath):
   global haz_pigz

   tar_params=["tar","xvf",tarfile,"--directory="+termpath]
   if haz_pigz:
      tar_params=tar_params+["--use-compress-program=pigz"]

   ret=subprocess.call(tar_params)
   if ret > 0:
      sys.stderr.write('***** TAR ERROR %s, command: %s *****\n' %
         (ret,tar_params))

def retrieve_tar_file(tmp_dir,container,obj_name,local_dir,prefix):
   global tar_suffix
   global root_id

   # download tar file and extract into terminal directory
   temp_file=unique_id()+tar_suffix
   if tmp_dir:
      temp_file=os.path.join(tmp_dir,temp_file)

   sw_download("--output="+temp_file,container,obj_name)

   # strip prefix and if next char is /, strip it too
   if prefix and obj_name.startswith(prefix):
      obj_name=obj_name[len(prefix):]
      if obj_name[0]=='/':
         obj_name=obj_name[1:]

   # if bundle, extract using tar embedded paths
   if obj_name.endswith(root_id+tar_suffix):
      term_path=local_dir
   else:
      term_path=create_local_path(local_dir,obj_name)

   extract_tar_file(temp_file,term_path)

   os.unlink(temp_file)

def extract_worker(item):
   retrieve_tar_file(item[0],item[1],item[2],item[3],item[4])

def extract_to_local(local_dir,container,no_hidden,tmp_dir,prefix,par):
   global tar_suffix
   global root_id

   swift_conn=create_sw_conn()
   if swift_conn:
      extract=[]

      try:
         headers,objs=swift_conn.get_container(container,prefix=prefix,
            full_listing=True)
         for obj in objs:
            if obj['name'].endswith(tar_suffix):
               if no_hidden and is_hidden_dir(obj['name']):
                  continue

               # param order: [tmp_dir,container,obj_name,local_dir,prefix]
               extract.append([tmp_dir,container,obj['name'],local_dir,prefix])
      except ClientException:
         print("Error: cannot access Swift container '%s'!" % container)

      extract_pool=multiprocessing.Pool(par)
      extract_pool.map(extract_worker,extract)

      swift_conn.close()

def usage():
   print("archive [parameters]")
   print("Parameters:")
   print("\t-l local_directory (default .)")
   print("\t-c container (required)")
   print("\t-x (extract from container to local directory)")
   print("\t-n (no hidden directories)")
   print("\t-t temp_dir (directory for temp files)")
   print("\t-a auth_token (default OS_AUTH_TOKEN)")
   print("\t-s storage_url (default OS_STORAGE_URL)")
   print("\t-p prefix")
   print("\t-P parallel_instances (default 3)")
   print("\t-m name:value (set container metadata)")

# is path a child of tree?
def is_subtree(tree,path):
   tree_sp=tree.split('/')
   path_sp=path.split('/')

   if len(path_sp)<len(tree_sp):
      return False

   for t,p in zip(tree_sp,path_sp):
      if t!=p:
         return False

   return True

def validate_dir(path,param):
   if not os.path.isdir(path):
      print("Error: %s '%s' is not accessible!" % (param,path))
      sys.exit()

   if path[-1]=='/':
      path=path[:-1]

   return(path)

def mywalk(top, skipdirs=['.snapshot',]):
    """ returns subset of os.walk  """
    for root, dirs, files in walk(top,topdown=True,onerror=walkerr):
        for skipdir in skipdirs:
            if skipdir in dirs:
                dirs.remove(skipdir)  # don't visit this directory
        yield root, dirs, files

def walkerr(oserr):
    sys.stderr.write(str(oserr))
    sys.stderr.write('\n')
    return 0

def main(argv):
   global swift_auth_token
   global storage_url
   global haz_pigz

   meta=[]
   sub_tree=""
   local_dir="."
   container=""
   tmp_dir=""
   extract=False
   no_hidden=False
   prefix=""
   par=3

   try:
      opts,args=getopt.getopt(argv,"l:c:t:a:s:p:P:S:m:xnh")
   except getopt.GetoptError:
      usage()
      sys.exit()

   for opt,arg in opts:
      if opt in ("-h"):
         usage()
         sys.exit()
      elif opt in ("-l"): # override default local directory
         local_dir=validate_dir(arg,"local")
      elif opt in ("-c"): # set container
         container=arg
      elif opt in ("-t"): # temp file directory
         tmp_dir=validate_dir(arg,"tmp_dir")
      elif opt in ("-a"): # override auth_token
         swift_auth_token=arg
      elif opt in ("-s"): # override storage URL
         storage_url=arg
      elif opt in ("-p"): # set prefix
         prefix=arg
      elif opt in ("-P"): # set parallel threads
         par=int(arg)
      elif opt in ("-x"): # extract mode
         extract=True
      elif opt in ("-n"): # set no-hidden flag to skip .*
         no_hidden=True
      elif opt in ("-S"): # specify optional sub-tree
         sub_tree=validate_dir(os.path.join(local_dir,arg),"subtree")
      elif opt in ("-m"): # specify container metadata
         if arg.count(':')==1:
            meta.append("-m"+arg)
         else:
            print("Error: metadata not in format key:value!")
            sys.exit()

   if not container:
      usage()
   else:
      if find_executable("pigz"):
         haz_pigz=True

      if extract:
         extract_to_local(local_dir,container,no_hidden,tmp_dir,prefix,par)
      else:
         archive_to_swift(local_dir,container,no_hidden,tmp_dir,prefix,par,
            sub_tree,meta)

if __name__=="__main__":
   main(sys.argv[1:])
<|endoftext|>


<commit_before>import os

import numpy as np
import pandas as pd
from attrdict import AttrDict

from ofm_helper.common_settings import TRANSFERS_DIR

class TransferFilter(AttrDict):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        if not hasattr(self, 'positions'):
            self.positions = None
        if not hasattr(self, 'ages'):
            self.ages = None
        if not hasattr(self, 'strengths'):
            self.strengths = None
        if not hasattr(self, 'seasons'):
            self.seasons = None
        if not hasattr(self, 'matchdays'):
            self.matchdays = None
        if not hasattr(self, 'min_price'):
            self.min_price = None
        if not hasattr(self, 'max_price'):
            self.max_price = None

class PandaManager:
    def __init__(self):
        self.data_frame = self._load_data()
        self.df_filterable_attributes = list(self.data_frame.columns.values)
        self.df_filterable_attributes.remove('Price')

    def get_data(self):
        if self.data_frame is None or self.data_frame.empty:
            self.data_frame = self._load_data()
        return self.data_frame

    def filter_transfers(self, transfer_filter=None):
        filtered_df = self.get_data().copy()

        if transfer_filter:
            for attribute in self.df_filterable_attributes:
                filtered_df = self._filter_for(filtered_df, transfer_filter, attribute)
            if transfer_filter.min_price:
                filtered_df = filtered_df[filtered_df.Price >= transfer_filter.min_price]
            if transfer_filter.max_price:
                filtered_df = filtered_df[filtered_df.Price <= transfer_filter.max_price]
            return filtered_df
        else:
            return self.data_frame

    @staticmethod
    def _filter_for(data_frame, transfer_filter, attribute):
        transfer_filter_attribute = str(attribute).lower()+'s'
        if transfer_filter[transfer_filter_attribute]:
            filtered_df = data_frame[data_frame[attribute].isin(transfer_filter[transfer_filter_attribute])]
            return filtered_df
        return data_frame

    def get_grouped_prices(self, group_by='Strength', **kwargs):
        df = self.filter_transfers(TransferFilter(**kwargs))
        return df.groupby(group_by).Price

    def _load_data(self):
        self.data_frame = pd.DataFrame()
        for file in os.listdir(TRANSFERS_DIR):
            if file.endswith('csv'):
                df = pd.read_csv(os.path.join(TRANSFERS_DIR, file),
                                 index_col=0,
                                 dtype={7: np.int32, 8: np.int32, 9: np.int32},
                                 skip_blank_lines=True,
                                 )
                df.drop(df.columns[[2, 3, 4]], axis=1, inplace=True)

                df = df.rename(columns={df.columns[0]: "Matchday",
                                        df.columns[1]: "Season",
                                        df.columns[2]: "Position",
                                        df.columns[3]: "Age",
                                        df.columns[4]: "Strength",
                                        df.columns[5]: "Price",
                                        })

                if self.data_frame.empty:
                    self.data_frame = df
                else:
                    self.data_frame = self.data_frame.append(df)

        return self.data_frame
<commit_msg>#19 split put filterable attributes in TransferFilter as well<commit_after>import os

import numpy as np
import pandas as pd
from attrdict import AttrDict

from ofm_helper.common_settings import TRANSFERS_DIR

class TransferFilter(AttrDict):
    def __init__(self, *args, **kwargs):
        filterable_attributes = ['positions', 'ages', 'strengths', 'seasons', 'matchdays', 'min_price', 'max_price']
        super().__init__(*args, **kwargs)
        for attribute in filterable_attributes:
            if not hasattr(self, attribute):
                self[attribute] = None

class PandaManager:
    def __init__(self):
        self.data_frame = self._load_data()
        self.df_filterable_attributes = list(self.data_frame.columns.values)
        self.df_filterable_attributes.remove('Price')

    def get_data(self):
        if self.data_frame is None or self.data_frame.empty:
            self.data_frame = self._load_data()
        return self.data_frame

    def filter_transfers(self, transfer_filter=None):
        filtered_df = self.get_data().copy()

        if transfer_filter:
            for attribute in self.df_filterable_attributes:
                filtered_df = self._filter_for(filtered_df, transfer_filter, attribute)
            if transfer_filter.min_price:
                filtered_df = filtered_df[filtered_df.Price >= transfer_filter.min_price]
            if transfer_filter.max_price:
                filtered_df = filtered_df[filtered_df.Price <= transfer_filter.max_price]
            return filtered_df
        else:
            return self.data_frame

    @staticmethod
    def _filter_for(data_frame, transfer_filter, attribute):
        transfer_filter_attribute = str(attribute).lower()+'s'
        if transfer_filter[transfer_filter_attribute]:
            filtered_df = data_frame[data_frame[attribute].isin(transfer_filter[transfer_filter_attribute])]
            return filtered_df
        return data_frame

    def get_grouped_prices(self, group_by='Strength', **kwargs):
        df = self.filter_transfers(TransferFilter(**kwargs))
        return df.groupby(group_by).Price

    def _load_data(self):
        self.data_frame = pd.DataFrame()
        for file in os.listdir(TRANSFERS_DIR):
            if file.endswith('csv'):
                df = pd.read_csv(os.path.join(TRANSFERS_DIR, file),
                                 index_col=0,
                                 dtype={7: np.int32, 8: np.int32, 9: np.int32},
                                 skip_blank_lines=True,
                                 )
                df.drop(df.columns[[2, 3, 4]], axis=1, inplace=True)

                df = df.rename(columns={df.columns[0]: "Matchday",
                                        df.columns[1]: "Season",
                                        df.columns[2]: "Position",
                                        df.columns[3]: "Age",
                                        df.columns[4]: "Strength",
                                        df.columns[5]: "Price",
                                        })

                if self.data_frame.empty:
                    self.data_frame = df
                else:
                    self.data_frame = self.data_frame.append(df)

        return self.data_frame
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#

import re
import httplib
import traceback
import time
from cyclone import web
from twisted.python.failure import Failure
from leela.server import logger
from leela.server.data import pp

class RestHandler(web.RequestHandler):

    no_keep_alive = True
    no_xsrf       = True

    def initialize(self, **kwargs):
        for (k, v) in kwargs.iteritems():
            setattr(self, k, v)

    def compute_etag(self):
        return(None)

    def _write_debug(self, chunk):
        if (isinstance(chunk, dict)):
            if (isinstance(chunk.get("debug"), dict)):
                debug = dict(chunk["debug"])
            else:
                debug = {}
            debug["uri"] = self.request.uri
            debug["now"] = time.strftime("%Y%m%dT%H%M%S%z")
            if (self.get_argument("debug", None) is None):
                if ("debug" in chunk):
                    del(chunk["debug"])
            else:
                chunk["debug"] = debug

    def write(self, chunk, cache=300):
        self._write_debug(chunk)
        cc  = self.get_argument("callback", "")
        bdy = pp.render_json(chunk)
        self.set_header("Cache-Control", "public, max-age=%d" % cache)
        if (re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", cc)):
            self.set_header("Content-Type", "text/javascript; charset=utf-8")
            bdy = u"%s(%s);" % (cc, bdy)
        else:
            self.set_header("Content-Type", "application/json")
        super(RestHandler, self).write(bdy)

    def write_error(self, status_code, **kwargs):
        debug  = kwargs.get("debug", {})
        if (not isinstance(debug, dict)):
            debug = {}
        if ("exception" in kwargs):
            e = kwargs["exception"]
            if (hasattr(e, "getErrorMessage")):
                debug["exception"] = e.getErrorMessage()
            else:
                debug["exception"] = str(e)
            if (hasattr(e, "getTraceback")):
                debug["stacktrace"] = e.getTraceback()
            else:
                debug["stacktrace"] = traceback.format_exc()
        rply = { "status": status_code,
                 "reason": httplib.responses[status_code],
                 "debug": debug
               }
        self.write(rply, 0)
        self.finish()

class Always404(RestHandler):

    def get(self):
        raise(web.HTTPError(404))

    def post(self):
        raise(web.HTTPError(404))

    def head(self):
        raise(web.HTTPError(404))

    def delete(self):
        raise(web.HTTPError(404))

    def options(self):
        raise(web.HTTPError(404))

def logexceptions(f):
    def g (*args, **kwargs):
        try:
            return f(*args, **kwargs)
        except:
            logger.exception()
            raise
    g.__name__ = f.__name__
    return g
<commit_msg>bugfix: using self.write_error instead of raise and write_error now sets the http status code properly<commit_after># -*- coding: utf-8; -*-
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#

import re
import httplib
import traceback
import time
from cyclone import web
from twisted.python.failure import Failure
from leela.server import logger
from leela.server.data import pp

class RestHandler(web.RequestHandler):

    no_keep_alive = True
    no_xsrf       = True

    def initialize(self, **kwargs):
        for (k, v) in kwargs.iteritems():
            setattr(self, k, v)

    def compute_etag(self):
        return(None)

    def _write_debug(self, chunk):
        if (isinstance(chunk, dict)):
            if (isinstance(chunk.get("debug"), dict)):
                debug = dict(chunk["debug"])
            else:
                debug = {}
            debug["uri"] = self.request.uri
            debug["now"] = time.strftime("%Y%m%dT%H%M%S%z")
            if (self.get_argument("debug", None) is None):
                if ("debug" in chunk):
                    del(chunk["debug"])
            else:
                chunk["debug"] = debug

    def write(self, chunk, cache=300):
        self._write_debug(chunk)
        cc  = self.get_argument("callback", "")
        bdy = pp.render_json(chunk)
        self.set_header("Cache-Control", "public, max-age=%d" % cache)
        if (re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", cc)):
            self.set_status(200)
            self.set_header("Content-Type", "text/javascript; charset=utf-8")
            bdy = u"%s(%s);" % (cc, bdy)
        else:
            self.set_header("Content-Type", "application/json")
        super(RestHandler, self).write(bdy)

    def write_error(self, status_code, **kwargs):
        debug  = kwargs.get("debug", {})
        if (not isinstance(debug, dict)):
            debug = {}
        if ("exception" in kwargs):
            e = kwargs["exception"]
            if (hasattr(e, "getErrorMessage")):
                debug["exception"] = e.getErrorMessage()
            else:
                debug["exception"] = str(e)
            if (hasattr(e, "getTraceback")):
                debug["stacktrace"] = e.getTraceback()
            else:
                debug["stacktrace"] = traceback.format_exc()
        self.set_status(status_code)
        rply = { "status": status_code,
                 "reason": httplib.responses[status_code],
                 "debug": debug
               }
        self.write(rply, 0)
        self.finish()

class Always404(RestHandler):

    def get(self):
        self.write_error(404)

    def post(self):
        self.write_error(404)

    def head(self):
        self.write_error(404)

    def delete(self):
        self.write_error(404)

def logexceptions(f):
    def g (*args, **kwargs):
        try:
            return f(*args, **kwargs)
        except:
            logger.exception()
            raise
    g.__name__ = f.__name__
    return g
<|endoftext|>


<commit_before>import tensorflow as tf
from tensorflow.python.training import moving_averages

FLAGS = tf.app.flags.FLAGS

def _variable_on_cpu(name, shape, initializer, trainable = True):
    """Helper to create a Variable stored on CPU memory.

    Args:
            name: name of the variable
            shape: list of ints
            initializer: initializer for Variable

    Returns:
            Variable Tensor
    """
    with tf.device('/cpu:0'):
        var = tf.get_variable(name, shape, initializer=initializer, trainable = trainable)
    return var

def _variable_with_weight_decay(name, shape, wd = 0.0):
    """Helper to create an initialized Variable with weight decay.

    Note that the Variable is initialized with a xavier initialization.
    A weight decay is added only if one is specified.

    #Args:
            name: name of the variable
            shape: list of ints
            wd: add L2Loss weight decay multiplied by this float. If None, weight
                    decay is not added for this Variable.

    Returns:
            Variable Tensor
    """
    var = _variable_on_cpu(name, shape, tf.contrib.layers.xavier_initializer())
    # print("change var")
    # var = tf.Variable(tf.truncated_normal(shape, mean= 0.0, stddev = 1.0), name = name)
    if wd != 0.0:
        weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
        tf.add_to_collection('losses', weight_decay)
    return var

def _conv2d(x, w, b, strides, padding):
    return tf.nn.bias_add(tf.nn.conv2d(x, w,strides=strides, padding = padding), b)

def _conv3d(x, w, b, strides = [1,1,1,1,1], padding = 'SAME'):
    return tf.nn.bias_add(tf.nn.conv3d(x, w,strides=strides, padding = padding), b)

def add_leaky_relu(hl_tensor, leaky_param):
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _deconv2d(x, w, b, output_shape, strides, padding):
    return tf.nn.bias_add(tf.nn.conv2d_transpose(x, w, output_shape, strides, padding), b)

def _add_leaky_relu(hl_tensor, leaky_param):
    """ add leaky relu layer
        Args:
            leaky_params should be from 0.01 to 0.1
    """
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _max_pool(x, ksize, strides):
    """ 2d pool layer"""
    pool = tf.nn.max_pool(x, ksize=ksize, strides= strides,
            padding='VALID')
    return pool

def _max_pool3(x, ksize, strides, name):
    """ 3d pool layer"""
    pool = tf.nn.max_pool3d(x, ksize=ksize, strides= strides,
        padding='VALID', name = name)
    return pool

def _avg_pool3(x, ksize, strides, name):
    """ 3d average pool layer """
    pool = tf.nn.avg_pool3d(x, ksize = ksize, strides = strides,
            padding = 'VALID', name = name)
    return pool

def _batch_norm(inputs, decay = 0.999, center = True, scale = False, epsilon = 0.001,
				moving_vars = 'moving_vars', activation = None, is_training = None,
				trainable = True, restore = True, scope = None, reuse = None):
    """ Copied from slim/ops.py
        Adds a Batch Normalization layer.
        Args:

            inputs: a tensor of size [batch_size, height, width, channels]
                    or [batch_size, channels].
            decay: decay for the moving average.
            center: If True, subtract beta. If False, beta is not created and ignored.
            scale: If True, multiply by gamma. If False, gamma is
                    not used. When the next layer is linear (also e.g. ReLU), this can be
                    disabled since the scaling can be done by the next layer.
            epsilon: small float added to variance to avoid dividing by zero.
            moving_vars: collection to store the moving_mean and moving_variance.
            activation: activation function.
            is_training: a placeholder whether or not the model is in training mode.
            trainable: whether or not the variables should be trainable or not.
            restore: whether or not the variables should be marked for restore.
            scope: Optional scope for variable_op_scope.
            reuse: whether or not the layer and its variables should be reused. To be
                            able to reuse the layer scope must be given.

        Returns:
            a tensor representing the output of the operation.
    """
    inputs_shape = inputs.get_shape()
    with tf.variable_op_scope([inputs], scope, 'BatchNorm', reuse = reuse):
        axis = list(range(len(inputs_shape) - 1))
        params_shape = inputs_shape[-1:]
        beta, gamma = None, None

        if center:
                beta = _variable_on_cpu('beta', params_shape, tf.zeros_initializer)
        if scale:
                gamma = _variable_on_cpu('gamma', params_shape, tf.ones_initializer)

        # moving_collections = [moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES]
        moving_mean = _variable_on_cpu('moving_mean', params_shape,tf.zeros_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_mean)
        moving_variance = _variable_on_cpu('moving_variance', params_shape, tf.ones_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_variance)

        def train_phase():
            mean, variance = tf.nn.moments(inputs, axis)
            update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay)
            update_moving_variance = moving_averages.assign_moving_average(moving_variance,
                                                            variance, decay)
            with tf.control_dependencies([update_moving_mean, update_moving_variance]):
                return tf.identity(mean), tf.identity(variance)

        def test_phase():
            return moving_mean, moving_variance

        mean, variance = tf.cond(is_training, train_phase, test_phase)
        outputs = tf.nn.batch_normalization(inputs, mean, variance, beta, gamma, epsilon)
        outputs.set_shape(inputs.get_shape())

        if activation:
            outputs = activation(outputs)

        return outputs

def triplet_loss(infer, labels, radius = 2.0):
    """
    Args:
        infer: inference concatenate together with 2 * batch_size
        labels: 0 or 1 with batch_size
        radius:
    Return:
        loss: triplet loss
    """

    feature_1, feature_2 = tf.split(0,2,infer)

    feature_diff = tf.reduce_sum(tf.square(feature_1 - feature_2), 1)
    feature_list = tf.dynamic_partition(feature_diff, labels, 2)

    pos_list = feature_list[1]
    neg_list  = (tf.maximum(0.0, radius * radius - feature_list[0]))
    full_list = tf.concat(0,[pos_list, neg_list])
    loss = tf.reduce_mean(full_list)

    return loss

def l1_reg(input_tensor, weights):
    l1_reg_loss = tf.mul(tf.reduce_sum(tf.abs(input_tensor)), weights, name = "l1_reg_loss")
    tf.add_to_collection('losses', l1_reg_loss)

def l2_loss(infer, label, loss_type, layer_name):
    """
    Args:
        loss_type: 'SUM', 'MEAN'
            'SUM' uses reduce_sum
            'MEAN' uses reduce_mean
    """
    assert(loss_type == 'SUM' or loss_type == 'MEAN')
    with tf.variable_scope(layer_name):
        if loss_type == 'SUM':
            loss = tf.reduce_sum(tf.square(infer - label))
        else:
            loss = tf.reduce_mean(tf.square(infer - label))

    return loss

def convolution_2d_layer(x, kernel_shape, kernel_stride, padding, wd, layer_name):
    """
    Args:
        x
        kernel_shape: [height, width, input_channel, output_channel]
        kernel_stride: [height, width]
        padding: "SAME" or "VALID"
        wd: weight decay params
        layer_name:
    """
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-1]], tf.constant_initializer(0.0))
        conv = _conv2d(x, weights, biases, [1, kernel_stride[0], kernel_stride[1],1], padding)

    return conv

def fully_connected_layer(x, output_num, wd, layer_name):
    """
    Args:
        x
        output_num
        wd
        layer_num
    """
    input_shape = x.get_shape().as_list()
    if len(input_shape) > 2:
        x = tf.reshape(x, [input_shape[0], -1])

    input_shape = x.get_shape().as_list()

    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay("weights", [input_shape[1], output_num], wd)
        biases = _variable_on_cpu('biases', [output_num], tf.constant_initializer(0.0))
        fc = tf.matmul(x, weights) + biases
    return fc

def deconvolution_2d_layer(x, kernel_shape, kernel_stride, output_shape, padding, wd, layer_name):
    """
    Args:
        x
        kernel_shape: [height, width, output_channel, input_channel]
        kernel_stride: [height, width]
        output_shape: [batch_size, height, width, channel]
        padding: "SAME" or "VALID"
        wd: weight decay params
        layer_name:
    """
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-2]], tf.constant_initializer(0.0))
        deconv = _deconv2d(x, weights, biases, output_shape, [1, kernel_stride[0], kernel_stride[1], 1], padding)
    return deconv

def maxpool_2d_layer(x, kernel_shape, kernel_stride, layer_name):
    """
    Args:
        x
        kernel_shape: [height, weights]
        kernel_stride: [height, weights]
    """

    with tf.variable_scope(layer_name):
        max_pool = _max_pool(x, [1, kernel_shape[0], kernel_shape[1], 1], [1, kernel_stride[0], kernel_stride[1], 1])
    return max_pool

def res_layer(x, kernel_shape, kernel_stride, padding, wd, layer_name, repeat_num, leaky_param = 0.01, is_train = None):
    """
    Args:
        x
        kernel_shape: [height, weights, input_channel, ouput_channel]
        kernel_stride: [height, weights]
        padding: SAME or VALID
        is_train: a tensor indicate is train or not
    """
    with tf.variable_scope(layer_name):
        conv = tf.identity(x)
        for i in xrange(repeat_num):
            conv = convolution_2d_layer(conv, kernel_shape,
                        kernel_stride, padding, wd, "_%d"%i)
            if is_train is not None:
                conv = _batch_norm(conv, is_training = is_train)
            conv = add_leaky_relu(conv, leaky_param)

        final_conv = tf.add(conv,x, 'res_connect')
    return final_conv

def res_pad(x, input_channel, output_channel, layer_name):
    """
    Args:
        x
        input_channel: a number
        output_channel: a number
        layer_name
    """
    with tf.variable_scope(layer_name):
        forward_pad = (output_channel - input_channel) // 2
        backward_pad = output_channel - input_channel - forward_pad
        x_pad = tf.pad(x, [[0,0],[0,0],[0,0],[forward_pad, backward_pad]])
    return x_pad

def copy_layer(x, layer_handle, repeat_num, layer_name, *params):
    """
    Args:
        x
        layer_handle: function handler
        repeat_num: the number of repeat
        layer_name
        params: parameters for the function
    """
    for i in xrange(repeat_num):
        with tf.variable_scope(layer_name + "_%d"%i):
            x = layer_handle(x, *params)
    return x

def unpooling_layer(x, output_size, layer_name):
    """ Bilinear Interpotation resize
    Args:
        x
        output_size [image_height, image_width]
        layer_name
    """
    with tf.variable_scope(layer_name):
        return tf.image.resize_images(x, output_size[0], output_size[1])

<commit_msg>add huber loss<commit_after>import tensorflow as tf
from tensorflow.python.training import moving_averages

FLAGS = tf.app.flags.FLAGS

def _variable_on_cpu(name, shape, initializer, trainable = True):
    """Helper to create a Variable stored on CPU memory.

    Args:
            name: name of the variable
            shape: list of ints
            initializer: initializer for Variable

    Returns:
            Variable Tensor
    """
    with tf.device('/cpu:0'):
        var = tf.get_variable(name, shape, initializer=initializer, trainable = trainable)
    return var

def _variable_with_weight_decay(name, shape, wd = 0.0):
    """Helper to create an initialized Variable with weight decay.

    Note that the Variable is initialized with a xavier initialization.
    A weight decay is added only if one is specified.

    #Args:
            name: name of the variable
            shape: list of ints
            wd: add L2Loss weight decay multiplied by this float. If None, weight
                    decay is not added for this Variable.

    Returns:
            Variable Tensor
    """
    var = _variable_on_cpu(name, shape, tf.contrib.layers.xavier_initializer())
    # print("change var")
    # var = tf.Variable(tf.truncated_normal(shape, mean= 0.0, stddev = 1.0), name = name)
    if wd != 0.0:
        weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
        tf.add_to_collection('losses', weight_decay)
    return var

def _conv2d(x, w, b, strides, padding):
    return tf.nn.bias_add(tf.nn.conv2d(x, w,strides=strides, padding = padding), b)

def _conv3d(x, w, b, strides = [1,1,1,1,1], padding = 'SAME'):
    return tf.nn.bias_add(tf.nn.conv3d(x, w,strides=strides, padding = padding), b)

def add_leaky_relu(hl_tensor, leaky_param):
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _deconv2d(x, w, b, output_shape, strides, padding):
    return tf.nn.bias_add(tf.nn.conv2d_transpose(x, w, output_shape, strides, padding), b)

def _add_leaky_relu(hl_tensor, leaky_param):
    """ add leaky relu layer
        Args:
            leaky_params should be from 0.01 to 0.1
    """
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _max_pool(x, ksize, strides):
    """ 2d pool layer"""
    pool = tf.nn.max_pool(x, ksize=ksize, strides= strides,
            padding='VALID')
    return pool

def _max_pool3(x, ksize, strides, name):
    """ 3d pool layer"""
    pool = tf.nn.max_pool3d(x, ksize=ksize, strides= strides,
        padding='VALID', name = name)
    return pool

def _avg_pool3(x, ksize, strides, name):
    """ 3d average pool layer """
    pool = tf.nn.avg_pool3d(x, ksize = ksize, strides = strides,
            padding = 'VALID', name = name)
    return pool

def _batch_norm(inputs, decay = 0.999, center = True, scale = False, epsilon = 0.001,
				moving_vars = 'moving_vars', activation = None, is_training = None,
				trainable = True, restore = True, scope = None, reuse = None):
    """ Copied from slim/ops.py
        Adds a Batch Normalization layer.
        Args:

            inputs: a tensor of size [batch_size, height, width, channels]
                    or [batch_size, channels].
            decay: decay for the moving average.
            center: If True, subtract beta. If False, beta is not created and ignored.
            scale: If True, multiply by gamma. If False, gamma is
                    not used. When the next layer is linear (also e.g. ReLU), this can be
                    disabled since the scaling can be done by the next layer.
            epsilon: small float added to variance to avoid dividing by zero.
            moving_vars: collection to store the moving_mean and moving_variance.
            activation: activation function.
            is_training: a placeholder whether or not the model is in training mode.
            trainable: whether or not the variables should be trainable or not.
            restore: whether or not the variables should be marked for restore.
            scope: Optional scope for variable_op_scope.
            reuse: whether or not the layer and its variables should be reused. To be
                            able to reuse the layer scope must be given.

        Returns:
            a tensor representing the output of the operation.
    """
    inputs_shape = inputs.get_shape()
    with tf.variable_op_scope([inputs], scope, 'BatchNorm', reuse = reuse):
        axis = list(range(len(inputs_shape) - 1))
        params_shape = inputs_shape[-1:]
        beta, gamma = None, None

        if center:
                beta = _variable_on_cpu('beta', params_shape, tf.zeros_initializer)
        if scale:
                gamma = _variable_on_cpu('gamma', params_shape, tf.ones_initializer)

        # moving_collections = [moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES]
        moving_mean = _variable_on_cpu('moving_mean', params_shape,tf.zeros_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_mean)
        moving_variance = _variable_on_cpu('moving_variance', params_shape, tf.ones_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_variance)

        def train_phase():
            mean, variance = tf.nn.moments(inputs, axis)
            update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay)
            update_moving_variance = moving_averages.assign_moving_average(moving_variance,
                                                            variance, decay)
            with tf.control_dependencies([update_moving_mean, update_moving_variance]):
                return tf.identity(mean), tf.identity(variance)

        def test_phase():
            return moving_mean, moving_variance

        mean, variance = tf.cond(is_training, train_phase, test_phase)
        outputs = tf.nn.batch_normalization(inputs, mean, variance, beta, gamma, epsilon)
        outputs.set_shape(inputs.get_shape())

        if activation:
            outputs = activation(outputs)

        return outputs

def triplet_loss(infer, labels, radius = 2.0):
    """
    Args:
        infer: inference concatenate together with 2 * batch_size
        labels: 0 or 1 with batch_size
        radius:
    Return:
        loss: triplet loss
    """

    feature_1, feature_2 = tf.split(0,2,infer)

    feature_diff = tf.reduce_sum(tf.square(feature_1 - feature_2), 1)
    feature_list = tf.dynamic_partition(feature_diff, labels, 2)

    pos_list = feature_list[1]
    neg_list  = (tf.maximum(0.0, radius * radius - feature_list[0]))
    full_list = tf.concat(0,[pos_list, neg_list])
    loss = tf.reduce_mean(full_list)

    return loss

def l1_reg(input_tensor, weights):
    l1_reg_loss = tf.mul(tf.reduce_sum(tf.abs(input_tensor)), weights, name = "l1_reg_loss")
    tf.add_to_collection('losses', l1_reg_loss)

def l2_loss(infer, label, loss_type, layer_name):
    """
    Args:
        loss_type: 'SUM', 'MEAN'
            'SUM' uses reduce_sum
            'MEAN' uses reduce_mean
    """
    assert(loss_type == 'SUM' or loss_type == 'MEAN')
    with tf.variable_scope(layer_name):
        if loss_type == 'SUM':
            loss = tf.reduce_sum(tf.square(infer - label))
        else:
            loss = tf.reduce_mean(tf.square(infer - label))

    return loss

def huber_loss(infer, label, epsilon, layer_name):
    """
    Args:
        infer
        label
        epsilon
        layer_name
    """
    with tf.variable_scope(layer_name):
        index = tf.to_int32(tf.abs(infer - label) > epsilon, name = 'partition_index')
        l1_part, l2_part = tf.dynamic_partition(tf.sub(infer,label), index, 2)
        l1_loss = tf.reduce_mean(tf.abs(l1_part), 'l1_loss')
        l2_loss = tf.reduce_mean(tf.abs(l2_part), 'l2_loss')
        hloss = tf.add(l1_loss, l2_loss, name = 'huber_loss_sum')
    return hloss

def convolution_2d_layer(x, kernel_shape, kernel_stride, padding, wd, layer_name):
    """
    Args:
        x
        kernel_shape: [height, width, input_channel, output_channel]
        kernel_stride: [height, width]
        padding: "SAME" or "VALID"
        wd: weight decay params
        layer_name:
    """
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-1]], tf.constant_initializer(0.0))
        conv = _conv2d(x, weights, biases, [1, kernel_stride[0], kernel_stride[1],1], padding)

    return conv

def fully_connected_layer(x, output_num, wd, layer_name):
    """
    Args:
        x
        output_num
        wd
        layer_num
    """
    input_shape = x.get_shape().as_list()
    if len(input_shape) > 2:
        x = tf.reshape(x, [input_shape[0], -1])

    input_shape = x.get_shape().as_list()

    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay("weights", [input_shape[1], output_num], wd)
        biases = _variable_on_cpu('biases', [output_num], tf.constant_initializer(0.0))
        fc = tf.matmul(x, weights) + biases
    return fc

def deconvolution_2d_layer(x, kernel_shape, kernel_stride, output_shape, padding, wd, layer_name):
    """
    Args:
        x
        kernel_shape: [height, width, output_channel, input_channel]
        kernel_stride: [height, width]
        output_shape: [batch_size, height, width, channel]
        padding: "SAME" or "VALID"
        wd: weight decay params
        layer_name:
    """
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-2]], tf.constant_initializer(0.0))
        deconv = _deconv2d(x, weights, biases, output_shape, [1, kernel_stride[0], kernel_stride[1], 1], padding)
    return deconv

def maxpool_2d_layer(x, kernel_shape, kernel_stride, layer_name):
    """
    Args:
        x
        kernel_shape: [height, weights]
        kernel_stride: [height, weights]
    """

    with tf.variable_scope(layer_name):
        max_pool = _max_pool(x, [1, kernel_shape[0], kernel_shape[1], 1], [1, kernel_stride[0], kernel_stride[1], 1])
    return max_pool

def res_layer(x, kernel_shape, kernel_stride, padding, wd, layer_name, repeat_num, leaky_param = 0.01, is_train = None):
    """
    Args:
        x
        kernel_shape: [height, weights, input_channel, ouput_channel]
        kernel_stride: [height, weights]
        padding: SAME or VALID
        is_train: a tensor indicate is train or not
    """
    with tf.variable_scope(layer_name):
        conv = tf.identity(x)
        for i in xrange(repeat_num):
            conv = convolution_2d_layer(conv, kernel_shape,
                        kernel_stride, padding, wd, "_%d"%i)
            if is_train is not None:
                conv = _batch_norm(conv, is_training = is_train)
            conv = add_leaky_relu(conv, leaky_param)

        final_conv = tf.add(conv,x, 'res_connect')
    return final_conv

def res_pad(x, input_channel, output_channel, layer_name):
    """
    Args:
        x
        input_channel: a number
        output_channel: a number
        layer_name
    """
    with tf.variable_scope(layer_name):
        forward_pad = (output_channel - input_channel) // 2
        backward_pad = output_channel - input_channel - forward_pad
        x_pad = tf.pad(x, [[0,0],[0,0],[0,0],[forward_pad, backward_pad]])
    return x_pad

def copy_layer(x, layer_handle, repeat_num, layer_name, *params):
    """
    Args:
        x
        layer_handle: function handler
        repeat_num: the number of repeat
        layer_name
        params: parameters for the function
    """
    for i in xrange(repeat_num):
        with tf.variable_scope(layer_name + "_%d"%i):
            x = layer_handle(x, *params)
    return x

def unpooling_layer(x, output_size, layer_name):
    """ Bilinear Interpotation resize
    Args:
        x
        output_size [image_height, image_width]
        layer_name
    """
    with tf.variable_scope(layer_name):
        return tf.image.resize_images(x, output_size[0], output_size[1])

<|endoftext|>