<commit_before>#############################################################################
# Copyright (C) 2007-2013 German Aerospace Center (DLR/SC)
#
# Created: 2010-08-13 Arne Bachmann <Arne.Bachmann@dlr.de>
# Changed: $Id: dbms.h 4577 20xx-xx-xx 09:27:39Z litz_ma $
#
# Version: $Revision: 3978 $
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#############################################################################

import sys
from ctypes import *

%s# generated enums are placed before this text

class TiglException(Exception):
    ''' The exception encapsulates the error return code of the library and arguments that were provided for the function. '''
    def __init__(self, code, *args, **kwargs):
        Exception.__init__(self)
        self.code = code
        if "error" in kwargs:
            self.error = str(kwargs["error"])
        elif code in TiglReturnCode._names:
            self.error = TiglReturnCode._names[code]
        else:
            self.error = "UNDEFINED"
        self.args = tuple(args)
        self.kwargs = dict(kwargs)
    def __str__(self):
        return self.error + " (" + str(self.code) + ") " + str(list(self.args)) + " " + str(self.kwargs)

class Tigl(object):
    def __init__(self):
        ''' The constructor initializes the TIGL library '''
        self._handle = c_int(-1)

        if sys.platform == 'win32':
            self.TIGL = cdll.TIGL
        elif sys.platform == "darwin":
            self.TIGL = CDLL("libTIGL.dylib")
        else:
            self.TIGL = CDLL("libTIGL.so")
        self.TIGL.tiglGetVersion.restype = c_char_p
        self.version = self.TIGL.tiglGetVersion()

    def __del__(self):
        ''' The destructor cleans up the library '''
        if hasattr(self, "TIGL"):
            if self.TIGL != None:
                self.close()
                self.TIGL = None

    def _validateReturnValue(self, tiglReturn, *args, **kwargs):
        ''' Helper function to raise an exception if return value is not TIGL_SUCCESS '''
        if tiglReturn != TiglReturnCode.TIGL_SUCCESS:
            raise TiglException(tiglReturn, args, kwargs)

    def open(self, tixi, configuration):
        ''' Open a document from a file. Tixi is from tixiwrapper and should have already opened a data set. '''
        if tixi._handle.value == -1:
            self._validateReturnValue(TiglReturnCode.TIGL_OPEN_FAILED) # raise a generic exception
        uuid = c_char_p(configuration)
        tiglReturn = self.TIGL.tiglOpenCPACSConfiguration(tixi._handle, uuid, byref(self._handle))
        if tiglReturn != TiglReturnCode.TIGL_SUCCESS:
            self._handle.value = -1
        self._validateReturnValue(tiglReturn, configuration)

    def close(self):
        ''' Closes the current document.
            Test cases contained in other functions already.
        '''
        if self._handle.value != -1:
            tiglReturn = self.TIGL.tiglCloseCPACSConfiguration(self._handle)
            self._handle.value = -1
            self._validateReturnValue(tiglReturn)

%s# the method definitions are placed above<commit_msg>python wrapper now does basic version checking.<commit_after>#############################################################################
# Copyright (C) 2007-2013 German Aerospace Center (DLR/SC)
#
# Created: 2010-08-13 Arne Bachmann <Arne.Bachmann@dlr.de>
# Changed: $Id: dbms.h 4577 20xx-xx-xx 09:27:39Z litz_ma $
#
# Version: $Revision: 3978 $
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#############################################################################

import sys
from ctypes import *

%s# generated enums are placed before this text

class TiglException(Exception):
    ''' The exception encapsulates the error return code of the library and arguments that were provided for the function. '''
    def __init__(self, code, *args, **kwargs):
        Exception.__init__(self)
        self.code = code
        if "error" in kwargs:
            self.error = str(kwargs["error"])
        elif code in TiglReturnCode._names:
            self.error = TiglReturnCode._names[code]
        else:
            self.error = "UNDEFINED"
        self.args = tuple(args)
        self.kwargs = dict(kwargs)
    def __str__(self):
        return self.error + " (" + str(self.code) + ") " + str(list(self.args)) + " " + str(self.kwargs)

class Tigl(object):
    def __init__(self):
        ''' The constructor initializes the TIGL library '''
        self._handle = c_int(-1)

        # We only support python2.5 - 3.0 now
        if sys.version_info>(3,0,0):
            print("Python3 not supported in tiglWrapper.")
            sys.exit()
        elif sys.version_info<(2,5,0):
            print("At least python 2.5 is needed from tiglWrapper.")
            sys.exit()

        if sys.platform == 'win32':
            self.TIGL = cdll.TIGL
        elif sys.platform == "darwin":
            self.TIGL = CDLL("libTIGL.dylib")
        else:
            self.TIGL = CDLL("libTIGL.so")
        self.TIGL.tiglGetVersion.restype = c_char_p
        self.version = self.TIGL.tiglGetVersion()

    def __del__(self):
        ''' The destructor cleans up the library '''
        if hasattr(self, "TIGL"):
            if self.TIGL != None:
                self.close()
                self.TIGL = None

    def _validateReturnValue(self, tiglReturn, *args, **kwargs):
        ''' Helper function to raise an exception if return value is not TIGL_SUCCESS '''
        if tiglReturn != TiglReturnCode.TIGL_SUCCESS:
            raise TiglException(tiglReturn, args, kwargs)

    def open(self, tixi, configuration):
        ''' Open a document from a file. Tixi is from tixiwrapper and should have already opened a data set. '''
        if tixi._handle.value == -1:
            self._validateReturnValue(TiglReturnCode.TIGL_OPEN_FAILED) # raise a generic exception
        uuid = c_char_p(configuration)
        tiglReturn = self.TIGL.tiglOpenCPACSConfiguration(tixi._handle, uuid, byref(self._handle))
        if tiglReturn != TiglReturnCode.TIGL_SUCCESS:
            self._handle.value = -1
        self._validateReturnValue(tiglReturn, configuration)

    def close(self):
        ''' Closes the current document.
            Test cases contained in other functions already.
        '''
        if self._handle.value != -1:
            tiglReturn = self.TIGL.tiglCloseCPACSConfiguration(self._handle)
            self._handle.value = -1
            self._validateReturnValue(tiglReturn)

%s# the method definitions are placed above<|endoftext|>


<commit_before>from astropy import wcs
from astropy import units as u
from astropy import constants
import warnings

def _parse_velocity_convention(vc):
    if vc in (u.doppler_radio, 'radio', 'RADIO', 'VRAD', 'F', 'FREQ'):
        return u.doppler_radio
    elif vc in (u.doppler_optical, 'optical', 'OPTICAL', 'VOPT', 'W', 'WAVE'):
        return u.doppler_optical
    elif vc in (u.doppler_relativistic, 'relativistic', 'RELATIVE', 'VREL',
                'speed', 'V', 'VELO'):
        return u.doppler_relativistic

# These are the only linear transformations allowed
LINEAR_CTYPES = {u.doppler_optical: 'VOPT', u.doppler_radio: 'VRAD',
                 u.doppler_relativistic: 'VELO'}
LINEAR_CTYPE_CHARS = {u.doppler_optical: 'W', u.doppler_radio: 'F',
                      u.doppler_relativistic: 'V'}

ALL_CTYPES = {'speed': LINEAR_CTYPES,
              'frequency': 'FREQ',
              'length': 'WAVE'}

CTYPE_TO_PHYSICALTYPE = {'WAVE': 'length',
                         'AIR': 'air wavelength', # unsupported
                         'FREQ': 'frequency',
                         'VELO': 'speed',
                         'VRAD': 'speed',
                         'VOPT': 'speed',
                         }

CTYPE_CHAR_TO_PHYSICALTYPE = {'W': 'length',
                              'A': 'air wavelength', # unsupported
                              'F': 'frequency',
                              'V': 'speed'}

CTYPE_TO_PHYSICALTYPE.update(CTYPE_CHAR_TO_PHYSICALTYPE)

PHYSICAL_TYPE_TO_CTYPE = dict([(v,k) for k,v in
                               CTYPE_CHAR_TO_PHYSICALTYPE.iteritems()])

# Used to indicate the intial / final sampling system
WCS_UNIT_DICT = {'F': u.Hz, 'W': u.m, 'V': u.m/u.s}
PHYS_UNIT_DICT = {'length': u.m, 'frequency': u.Hz, 'speed': u.m/u.s}

LINEAR_CUNIT_DICT = {'VRAD': u.Hz, 'VOPT': u.m}
LINEAR_CUNIT_DICT.update(WCS_UNIT_DICT)

def _get_linear_equivalency(unit1, unit2):
    """
    Determin the default / "natural" convention
    Radio <-> freq
    Optical <-> wavelength
    """

    if unit1.is_equivalent(unit2):
        # Only a change of units is needed
        return lambda *args: []
    elif unit1.is_equivalent(unit2, u.spectral()):
        # wavelength <-> frequency
        # this is NOT a linear transformation (w = 1/v)
        return None
    elif (unit1.physical_type in ('frequency','speed') and
          unit2.physical_type in ('frequency','speed')):
        # top 'if' statement rules out both being the same
        return u.doppler_radio
    elif (unit1.physical_type in ('length','speed') and
          unit2.physical_type in ('length','speed')):
        return u.doppler_optical

def determine_ctype_from_vconv(ctype, unit, velocity_convention=None):
    """
    Given a CTYPE describing the current WCS and an output unit and velocity
    convention, determine the appropriate output CTYPE

    Examples
    --------
    >>> determine_ctype_from_vconv('VELO-F2V', u.Hz)
    'FREQ'
    >>> determine_ctype_from_vconv('VELO-F2V', u.m)
    'WAVE-F2W'
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s)
    ValueError('A velocity convention must be specified')
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s, velocity_convention=u.doppler_radio)
    'VRAD'
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s, velocity_convention=u.doppler_optical)
    'VOPT-F2W'
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s, velocity_convention=u.doppler_relativistic)
    'VELO-F2V'
    """
    unit = u.Unit(unit)

    in_physchar = (ctype[0]if len(ctype)<=4 else
                   ctype[5])

    if unit.physical_type == 'speed':
        if velocity_convention is None:
            raise ValueError('A velocity convention must be specified')
        vcin = _parse_velocity_convention(ctype[:4])
        vcout = _parse_velocity_convention(velocity_convention)
        if vcin == vcout:
            return LINEAR_CTYPES[vcout]
        else:
            return "{type}-{s1}2{s2}".format(type=LINEAR_CTYPES[vcout],
                                             s1=in_physchar,
                                             s2=LINEAR_CTYPE_CHARS[vcout])

    else:
        in_phystype = CTYPE_CHAR_TO_PHYSICALTYPE[in_physchar]
        if in_phystype == unit.physical_type:
            # Linear case
            return ALL_CTYPES[in_phystype]
        else:
            # Nonlinear case
            out_physchar = PHYSICAL_TYPE_TO_CTYPE[unit.physical_type]
            return "{type}-{s1}2{s2}".format(type=ALL_CTYPES[unit.physical_type],
                                             s1=in_physchar,
                                             s2=out_physchar)

def get_rest_value_from_wcs(mywcs):
    if mywcs.wcs.restfrq:
        ref_value = mywcs.wcs.restfrq*u.Hz
        return ref_value
    elif mywcs.wcs.restwav:
        ref_value = mywcs.wcs.restwav*u.m
        return ref_value

def convert_spectral_axis(mywcs, outunit, out_ctype, rest_value=None):
    """
    Convert a spectral axis from its unit to a specified out unit with a given output
    ctype

    Only VACUUM units are supported (not air)

    Process:
        1. Convert the input unit to its equivalent linear unit
        2. Convert the input linear unit to the output linear unit
        3. Convert the output linear unit to the output unit
    """
    outunit = u.Unit(outunit)
    inunit = u.Unit(mywcs.wcs.cunit[mywcs.wcs.spec])
    in_spec_ctype = mywcs.wcs.ctype[mywcs.wcs.spec]

    # If the input unit is not linearly sampled, its linear equivalent will be
    # the 8th character in the ctype, and the linearly-sampled ctype will be
    # the 6th character
    # e.g.: VOPT-F2V
    lin_ctype = (in_spec_ctype[7] if len(in_spec_ctype) > 4 else in_spec_ctype[:4])
    lin_cunit = (LINEAR_CUNIT_DICT[lin_ctype] if lin_ctype in LINEAR_CUNIT_DICT
                 else mywcs.wcs.cunit[mywcs.wcs.spec])
    in_vcequiv = _parse_velocity_convention(in_spec_ctype[:4])

    out_ctype_conv = out_ctype[7] if len(out_ctype) > 4 else out_ctype[:4]
    out_lin_cunit = (LINEAR_CUNIT_DICT[out_ctype_conv] if out_ctype_conv in
                     LINEAR_CUNIT_DICT else outunit)
    out_vcequiv = _parse_velocity_convention(out_ctype_conv)

    ref_value = None
    if outunit.physical_type == 'speed':
        if rest_value is None:
            rest_value = get_rest_value_from_wcs(mywcs)
            if rest_value is None:
                raise ValueError("If converting from wavelength/frequency to speed, "
                                 "a reference wavelength/frequency is required.")
            else:
                warnings.warn("Using WCS built-in rest frequency even though the "
                              "WCS system was originally {0}".format(in_spec_ctype))
        ref_value = rest_value.to(u.Hz, u.spectral())
    elif inunit.physical_type == 'speed':
        # The rest frequency and wavelength should be equivalent
        wcs_rv = get_rest_value_from_wcs(mywcs)
        if rest_value is not None:
            warnings.warn("Overriding the reference value specified in the WCS.")
            ref_value = rest_value
        elif wcs_rv is not None:
            ref_value = wcs_rv
        else:
            raise ValueError("If converting from speed to wavelength/frequency, "
                             "a reference wavelength/frequency is required.")

    # Load the input values
    crval_in = (mywcs.wcs.crval[mywcs.wcs.spec] * inunit)
    cdelt_in = (mywcs.wcs.cdelt[mywcs.wcs.spec] * inunit)

    # 1. Convert input to input, linear
    if in_vcequiv is not None and ref_value is not None:
        crval_lin1 = crval_in.to(lin_cunit, u.spectral() + in_vcequiv(ref_value))
    else:
        crval_lin1 = crval_in.to(lin_cunit, u.spectral())
    cdelt_lin1 = cdelt_derivative(crval_in,
                                  cdelt_in,
                                  # equivalent: inunit.physical_type
                                  intype=CTYPE_TO_PHYSICALTYPE[in_spec_ctype[:4]],
                                  outtype=CTYPE_TO_PHYSICALTYPE[lin_ctype],
                                  rest=ref_value,
                                  linear=True
                                  )

    # 2. Convert input, linear to output, linear
    if ref_value is None:
        if in_vcequiv is not None:
            pass # consider raising a ValueError here; not clear if this is valid
        crval_lin2 = crval_lin1.to(out_lin_cunit, u.spectral())
    else:
        # at this stage, the transition can ONLY be relativistic, because the V
        # frame (as a linear frame) is only defined as "apparent velocity"
        crval_lin2 = crval_lin1.to(out_lin_cunit, u.spectral() +
                                   u.doppler_relativistic(ref_value))

    # For cases like VRAD <-> FREQ and VOPT <-> WAVE, this will be linear too:
    linear_middle = in_vcequiv == out_vcequiv

    cdelt_lin2 = cdelt_derivative(crval_lin1, cdelt_lin1,
                                  intype=CTYPE_TO_PHYSICALTYPE[lin_ctype],
                                  outtype=CTYPE_TO_PHYSICALTYPE[out_ctype_conv],
                                  rest=ref_value,
                                  linear=linear_middle)

    # 3. Convert output, linear to output
    if out_vcequiv is not None and ref_value is not None:
        crval_out = crval_lin2.to(outunit, out_vcequiv(ref_value) + u.spectral())
        #cdelt_out = cdelt_lin2.to(outunit, out_vcequiv(ref_value) + u.spectral())
        cdelt_out = cdelt_derivative(crval_lin2,
                                     cdelt_lin2,
                                     intype=CTYPE_TO_PHYSICALTYPE[out_ctype_conv],
                                     outtype=outunit.physical_type,
                                     rest=ref_value,
                                     linear=True
                                     ).to(outunit)
    else:
        crval_out = crval_lin2.to(outunit, u.spectral())
        cdelt_out = cdelt_lin2.to(outunit, u.spectral())

    if crval_out.unit != cdelt_out.unit:
        # this should not be possible, but it's a sanity check
        raise ValueError("Conversion failed: the units of cdelt and crval don't match.")

    # A cdelt of 0 would be meaningless
    if cdelt_out.value == 0:
        raise ValueError("Conversion failed: the output CDELT would be 0.")

    newwcs = mywcs.deepcopy()
    newwcs.wcs.cdelt[newwcs.wcs.spec] = cdelt_out.value
    newwcs.wcs.cunit[newwcs.wcs.spec] = cdelt_out.unit.to_string(format='fits')
    newwcs.wcs.crval[newwcs.wcs.spec] = crval_out.value
    newwcs.wcs.ctype[newwcs.wcs.spec] = out_ctype
    if rest_value is not None:
        if rest_value.unit.physical_type == 'frequency':
            newwcs.wcs.restfrq = rest_value.to(u.Hz).value
        elif rest_value.unit.physical_type == 'length':
            newwcs.wcs.restwav = rest_value.to(u.m).value
        else:
            raise ValueError("Rest Value was specified, but not in frequency or length units")

    return newwcs

def cdelt_derivative(crval, cdelt, intype, outtype, linear=False, rest=None):
    if intype == outtype:
        return cdelt
    elif set((outtype,intype)) == set(('length','frequency')):
        # Symmetric equations!
        return (-constants.c / crval**2 * cdelt).to(PHYS_UNIT_DICT[outtype])
    elif outtype in ('frequency','length') and intype == 'speed':
        if linear:
            numer = cdelt * rest.to(PHYS_UNIT_DICT[outtype], u.spectral())
            denom = constants.c
        else:
            numer = cdelt * constants.c * rest.to(PHYS_UNIT_DICT[outtype], u.spectral())
            denom = (constants.c + crval)*(constants.c**2 - crval**2)**0.5
        if outtype == 'frequency':
            return (-numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
        else:
            return (numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
    elif outtype == 'speed' and intype in ('frequency','length'):

        if linear:
            numer = cdelt * constants.c
            denom = rest.to(PHYS_UNIT_DICT[intype], u.spectral())
        else:
            numer = 4 * constants.c * crval * rest.to(crval.unit, u.spectral())**2 * cdelt
            denom = (crval**2 + rest.to(crval.unit, u.spectral())**2)**2
        if intype == 'frequency':
            return (-numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
        else:
            return (numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
    else:
        raise ValueError("Invalid in/out frames")
<commit_msg>py3: items not iteritems<commit_after>from astropy import wcs
from astropy import units as u
from astropy import constants
import warnings

def _parse_velocity_convention(vc):
    if vc in (u.doppler_radio, 'radio', 'RADIO', 'VRAD', 'F', 'FREQ'):
        return u.doppler_radio
    elif vc in (u.doppler_optical, 'optical', 'OPTICAL', 'VOPT', 'W', 'WAVE'):
        return u.doppler_optical
    elif vc in (u.doppler_relativistic, 'relativistic', 'RELATIVE', 'VREL',
                'speed', 'V', 'VELO'):
        return u.doppler_relativistic

# These are the only linear transformations allowed
LINEAR_CTYPES = {u.doppler_optical: 'VOPT', u.doppler_radio: 'VRAD',
                 u.doppler_relativistic: 'VELO'}
LINEAR_CTYPE_CHARS = {u.doppler_optical: 'W', u.doppler_radio: 'F',
                      u.doppler_relativistic: 'V'}

ALL_CTYPES = {'speed': LINEAR_CTYPES,
              'frequency': 'FREQ',
              'length': 'WAVE'}

CTYPE_TO_PHYSICALTYPE = {'WAVE': 'length',
                         'AIR': 'air wavelength', # unsupported
                         'FREQ': 'frequency',
                         'VELO': 'speed',
                         'VRAD': 'speed',
                         'VOPT': 'speed',
                         }

CTYPE_CHAR_TO_PHYSICALTYPE = {'W': 'length',
                              'A': 'air wavelength', # unsupported
                              'F': 'frequency',
                              'V': 'speed'}

CTYPE_TO_PHYSICALTYPE.update(CTYPE_CHAR_TO_PHYSICALTYPE)

PHYSICAL_TYPE_TO_CTYPE = dict([(v,k) for k,v in
                               CTYPE_CHAR_TO_PHYSICALTYPE.items()])

# Used to indicate the intial / final sampling system
WCS_UNIT_DICT = {'F': u.Hz, 'W': u.m, 'V': u.m/u.s}
PHYS_UNIT_DICT = {'length': u.m, 'frequency': u.Hz, 'speed': u.m/u.s}

LINEAR_CUNIT_DICT = {'VRAD': u.Hz, 'VOPT': u.m}
LINEAR_CUNIT_DICT.update(WCS_UNIT_DICT)

def _get_linear_equivalency(unit1, unit2):
    """
    Determin the default / "natural" convention
    Radio <-> freq
    Optical <-> wavelength
    """

    if unit1.is_equivalent(unit2):
        # Only a change of units is needed
        return lambda *args: []
    elif unit1.is_equivalent(unit2, u.spectral()):
        # wavelength <-> frequency
        # this is NOT a linear transformation (w = 1/v)
        return None
    elif (unit1.physical_type in ('frequency','speed') and
          unit2.physical_type in ('frequency','speed')):
        # top 'if' statement rules out both being the same
        return u.doppler_radio
    elif (unit1.physical_type in ('length','speed') and
          unit2.physical_type in ('length','speed')):
        return u.doppler_optical

def determine_ctype_from_vconv(ctype, unit, velocity_convention=None):
    """
    Given a CTYPE describing the current WCS and an output unit and velocity
    convention, determine the appropriate output CTYPE

    Examples
    --------
    >>> determine_ctype_from_vconv('VELO-F2V', u.Hz)
    'FREQ'
    >>> determine_ctype_from_vconv('VELO-F2V', u.m)
    'WAVE-F2W'
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s)
    ValueError('A velocity convention must be specified')
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s, velocity_convention=u.doppler_radio)
    'VRAD'
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s, velocity_convention=u.doppler_optical)
    'VOPT-F2W'
    >>> determine_ctype_from_vconv('FREQ', u.m/u.s, velocity_convention=u.doppler_relativistic)
    'VELO-F2V'
    """
    unit = u.Unit(unit)

    in_physchar = (ctype[0]if len(ctype)<=4 else
                   ctype[5])

    if unit.physical_type == 'speed':
        if velocity_convention is None:
            raise ValueError('A velocity convention must be specified')
        vcin = _parse_velocity_convention(ctype[:4])
        vcout = _parse_velocity_convention(velocity_convention)
        if vcin == vcout:
            return LINEAR_CTYPES[vcout]
        else:
            return "{type}-{s1}2{s2}".format(type=LINEAR_CTYPES[vcout],
                                             s1=in_physchar,
                                             s2=LINEAR_CTYPE_CHARS[vcout])

    else:
        in_phystype = CTYPE_CHAR_TO_PHYSICALTYPE[in_physchar]
        if in_phystype == unit.physical_type:
            # Linear case
            return ALL_CTYPES[in_phystype]
        else:
            # Nonlinear case
            out_physchar = PHYSICAL_TYPE_TO_CTYPE[unit.physical_type]
            return "{type}-{s1}2{s2}".format(type=ALL_CTYPES[unit.physical_type],
                                             s1=in_physchar,
                                             s2=out_physchar)

def get_rest_value_from_wcs(mywcs):
    if mywcs.wcs.restfrq:
        ref_value = mywcs.wcs.restfrq*u.Hz
        return ref_value
    elif mywcs.wcs.restwav:
        ref_value = mywcs.wcs.restwav*u.m
        return ref_value

def convert_spectral_axis(mywcs, outunit, out_ctype, rest_value=None):
    """
    Convert a spectral axis from its unit to a specified out unit with a given output
    ctype

    Only VACUUM units are supported (not air)

    Process:
        1. Convert the input unit to its equivalent linear unit
        2. Convert the input linear unit to the output linear unit
        3. Convert the output linear unit to the output unit
    """
    outunit = u.Unit(outunit)
    inunit = u.Unit(mywcs.wcs.cunit[mywcs.wcs.spec])
    in_spec_ctype = mywcs.wcs.ctype[mywcs.wcs.spec]

    # If the input unit is not linearly sampled, its linear equivalent will be
    # the 8th character in the ctype, and the linearly-sampled ctype will be
    # the 6th character
    # e.g.: VOPT-F2V
    lin_ctype = (in_spec_ctype[7] if len(in_spec_ctype) > 4 else in_spec_ctype[:4])
    lin_cunit = (LINEAR_CUNIT_DICT[lin_ctype] if lin_ctype in LINEAR_CUNIT_DICT
                 else mywcs.wcs.cunit[mywcs.wcs.spec])
    in_vcequiv = _parse_velocity_convention(in_spec_ctype[:4])

    out_ctype_conv = out_ctype[7] if len(out_ctype) > 4 else out_ctype[:4]
    out_lin_cunit = (LINEAR_CUNIT_DICT[out_ctype_conv] if out_ctype_conv in
                     LINEAR_CUNIT_DICT else outunit)
    out_vcequiv = _parse_velocity_convention(out_ctype_conv)

    ref_value = None
    if outunit.physical_type == 'speed':
        if rest_value is None:
            rest_value = get_rest_value_from_wcs(mywcs)
            if rest_value is None:
                raise ValueError("If converting from wavelength/frequency to speed, "
                                 "a reference wavelength/frequency is required.")
            else:
                warnings.warn("Using WCS built-in rest frequency even though the "
                              "WCS system was originally {0}".format(in_spec_ctype))
        ref_value = rest_value.to(u.Hz, u.spectral())
    elif inunit.physical_type == 'speed':
        # The rest frequency and wavelength should be equivalent
        wcs_rv = get_rest_value_from_wcs(mywcs)
        if rest_value is not None:
            warnings.warn("Overriding the reference value specified in the WCS.")
            ref_value = rest_value
        elif wcs_rv is not None:
            ref_value = wcs_rv
        else:
            raise ValueError("If converting from speed to wavelength/frequency, "
                             "a reference wavelength/frequency is required.")

    # Load the input values
    crval_in = (mywcs.wcs.crval[mywcs.wcs.spec] * inunit)
    cdelt_in = (mywcs.wcs.cdelt[mywcs.wcs.spec] * inunit)

    # 1. Convert input to input, linear
    if in_vcequiv is not None and ref_value is not None:
        crval_lin1 = crval_in.to(lin_cunit, u.spectral() + in_vcequiv(ref_value))
    else:
        crval_lin1 = crval_in.to(lin_cunit, u.spectral())
    cdelt_lin1 = cdelt_derivative(crval_in,
                                  cdelt_in,
                                  # equivalent: inunit.physical_type
                                  intype=CTYPE_TO_PHYSICALTYPE[in_spec_ctype[:4]],
                                  outtype=CTYPE_TO_PHYSICALTYPE[lin_ctype],
                                  rest=ref_value,
                                  linear=True
                                  )

    # 2. Convert input, linear to output, linear
    if ref_value is None:
        if in_vcequiv is not None:
            pass # consider raising a ValueError here; not clear if this is valid
        crval_lin2 = crval_lin1.to(out_lin_cunit, u.spectral())
    else:
        # at this stage, the transition can ONLY be relativistic, because the V
        # frame (as a linear frame) is only defined as "apparent velocity"
        crval_lin2 = crval_lin1.to(out_lin_cunit, u.spectral() +
                                   u.doppler_relativistic(ref_value))

    # For cases like VRAD <-> FREQ and VOPT <-> WAVE, this will be linear too:
    linear_middle = in_vcequiv == out_vcequiv

    cdelt_lin2 = cdelt_derivative(crval_lin1, cdelt_lin1,
                                  intype=CTYPE_TO_PHYSICALTYPE[lin_ctype],
                                  outtype=CTYPE_TO_PHYSICALTYPE[out_ctype_conv],
                                  rest=ref_value,
                                  linear=linear_middle)

    # 3. Convert output, linear to output
    if out_vcequiv is not None and ref_value is not None:
        crval_out = crval_lin2.to(outunit, out_vcequiv(ref_value) + u.spectral())
        #cdelt_out = cdelt_lin2.to(outunit, out_vcequiv(ref_value) + u.spectral())
        cdelt_out = cdelt_derivative(crval_lin2,
                                     cdelt_lin2,
                                     intype=CTYPE_TO_PHYSICALTYPE[out_ctype_conv],
                                     outtype=outunit.physical_type,
                                     rest=ref_value,
                                     linear=True
                                     ).to(outunit)
    else:
        crval_out = crval_lin2.to(outunit, u.spectral())
        cdelt_out = cdelt_lin2.to(outunit, u.spectral())

    if crval_out.unit != cdelt_out.unit:
        # this should not be possible, but it's a sanity check
        raise ValueError("Conversion failed: the units of cdelt and crval don't match.")

    # A cdelt of 0 would be meaningless
    if cdelt_out.value == 0:
        raise ValueError("Conversion failed: the output CDELT would be 0.")

    newwcs = mywcs.deepcopy()
    newwcs.wcs.cdelt[newwcs.wcs.spec] = cdelt_out.value
    newwcs.wcs.cunit[newwcs.wcs.spec] = cdelt_out.unit.to_string(format='fits')
    newwcs.wcs.crval[newwcs.wcs.spec] = crval_out.value
    newwcs.wcs.ctype[newwcs.wcs.spec] = out_ctype
    if rest_value is not None:
        if rest_value.unit.physical_type == 'frequency':
            newwcs.wcs.restfrq = rest_value.to(u.Hz).value
        elif rest_value.unit.physical_type == 'length':
            newwcs.wcs.restwav = rest_value.to(u.m).value
        else:
            raise ValueError("Rest Value was specified, but not in frequency or length units")

    return newwcs

def cdelt_derivative(crval, cdelt, intype, outtype, linear=False, rest=None):
    if intype == outtype:
        return cdelt
    elif set((outtype,intype)) == set(('length','frequency')):
        # Symmetric equations!
        return (-constants.c / crval**2 * cdelt).to(PHYS_UNIT_DICT[outtype])
    elif outtype in ('frequency','length') and intype == 'speed':
        if linear:
            numer = cdelt * rest.to(PHYS_UNIT_DICT[outtype], u.spectral())
            denom = constants.c
        else:
            numer = cdelt * constants.c * rest.to(PHYS_UNIT_DICT[outtype], u.spectral())
            denom = (constants.c + crval)*(constants.c**2 - crval**2)**0.5
        if outtype == 'frequency':
            return (-numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
        else:
            return (numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
    elif outtype == 'speed' and intype in ('frequency','length'):

        if linear:
            numer = cdelt * constants.c
            denom = rest.to(PHYS_UNIT_DICT[intype], u.spectral())
        else:
            numer = 4 * constants.c * crval * rest.to(crval.unit, u.spectral())**2 * cdelt
            denom = (crval**2 + rest.to(crval.unit, u.spectral())**2)**2
        if intype == 'frequency':
            return (-numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
        else:
            return (numer/denom).to(PHYS_UNIT_DICT[outtype], u.spectral())
    else:
        raise ValueError("Invalid in/out frames")
<|endoftext|>


<commit_before># coding: utf-8

"""
The four adjacent digits in the 1000-digit number that have the greatest
product are 9 × 9 × 8 × 9 = 5832.

73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450

Find the thirteen adjacent digits in the 1000-digit number that have the
greatest product. What is the value of this product?
"""

import time

BIG_NUMBER = """
73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450
""".replace('\n', '').strip()

def oneliner():
    "TODO: write oneliner version of this thing"

def solve1():
    adjacent_digits = 13

    def product(digits):
        _product = 1
        for d in digits:
            _product *= int(d)
        return _product

    products = []
    for n in range(len(BIG_NUMBER)-adjacent_digits-1):
        products.append(
            product(BIG_NUMBER[n:n+adjacent_digits])
        )

    return max(products)

if __name__ == "__main__":

    def timeit(function):
        t1 = time.time()
        output = function()
        t2 = time.time()
        return output, t2-t1

    print timeit(solve1)
    print timeit(oneliner)
<commit_msg>8.py -- added oneliner<commit_after># coding: utf-8

"""
The four adjacent digits in the 1000-digit number that have the greatest
product are 9 × 9 × 8 × 9 = 5832.

73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450

Find the thirteen adjacent digits in the 1000-digit number that have the
greatest product. What is the value of this product?
"""

import time

BIG_NUMBER = """
73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450
""".replace('\n', '').strip()

def oneliner():
    return max(
        reduce(lambda x, y: int(x) * int(y),
               BIG_NUMBER[n:n+13])
        for n in range(len(BIG_NUMBER) - 13 - 1)
    )

def solve1():
    adjacent_digits = 13

    def product(digits):
        _product = 1
        for d in digits:
            _product *= int(d)
        return _product

    products = []
    for n in range(len(BIG_NUMBER)-adjacent_digits-1):
        products.append(
            product(BIG_NUMBER[n:n+adjacent_digits])
        )

    return max(products)

if __name__ == "__main__":

    def timeit(function):
        t1 = time.time()
        output = function()
        t2 = time.time()
        return output, t2-t1

    print timeit(solve1)
    print timeit(oneliner)
<|endoftext|>


<commit_before>import re
import traceback

from gevent.pywsgi import WSGIServer
from gevent.queue import Channel

from spinoff.actor import Actor
from spinoff.actor.events import Events, Error
from spinoff.actor.exceptions import Unhandled
from spinoff.util.logging import dbg
from spinoff.util.pattern_matching import ANY
from .werkzeug_wrappers import BaseRequest

class HttpServer(Actor):
    def pre_start(self, address, responders, default_content_type='text/html'):
        self.responders = responders
        self.default_content_type = default_content_type
        self.server = WSGIServer(address, self.handle_wsgi_request)
        self.server.start()

    def handle_wsgi_request(self, env, start_response):
        ch = Channel()
        req = Request(ch, env, start_response, default_content_type=self.default_content_type)
        self << ('handle', req)
        return response_stream(ch)

    def receive(self, msg):
        if ('handle', ANY) == msg:
            _, req = msg
            try:
                responder = self.get_responder(req.env['PATH_INFO'])
                if responder:
                    responder, args, kwargs = responder
                    self.spawn(RequestHandler.using(req, responder, args, kwargs))
                else:
                    req.start_response('404 Not Found', [('Content-Type', 'text/html')])
                    req.write('
404 Not Found
\n')
                    req.write('The page you tried to reach could not be found.\n')
                    req.close()
            except:
                _send_500(req)
                raise

        elif 'get-addr' == msg:
            self.reply(self.server.address)
        else:
            raise Unhandled

    def get_responder(self, path):
        for pattern, responder in self.responders:
            m = re.compile(pattern).match(path)
            if m:
                args = m.groups()
                kwargs = m.groupdict()
                return responder, args if not kwargs else (), kwargs

class RequestHandler(Actor):
    def run(self, req, responder, args, kwargs):
        try:
            self.responder = responder = self.spawn(responder.using(req, *args, **kwargs))
            self.error = None
            try:
                Events.subscribe(Error, self.check_error)  # XXX: it would be nicer if Events.subscribe accepted an actor ref to filter on
                responder.join()
                if not req.closed:
                    if self.error:
                        _send_500('
%s
' % (''.join(traceback.format_exception(type(self.error.exc), self.error.exc, self.error.tb)),))
                    req.close()
            finally:
                Events.unsubscribe(Error, self.check_error)
        except:
            _send_500(req)
            raise

    def check_error(self, error):
        if error.actor == self.responder:
            self.error = error

class _BREAK(object):
    def __repr__(self):
        return '_BREAK'
_BREAK = _BREAK()

def response_stream(ch):
    while True:
        val = ch.get()
        if val is _BREAK:
            break
        yield val

class Request(BaseRequest):
    def __init__(self, ch, env, start_response, default_content_type):
        self.ch = ch
        self.env = env
        self._response_started = False
        self._start_response = start_response
        self.default_content_type = default_content_type
        self.closed = False
        BaseRequest.__init__(self, env)

    def set_status(self, status):
        self.start_response(status, [('Content-Type', self.default_content_type)])

    def start_response(self, *args):
        self._response_started = True
        ret = self._start_response(*args)
        self.start_response = self._start_response  # optimization
        return ret

    def write(self, data):
        if not self._response_started:
            self._start_response('200 OK', [('Content-Type', 'text/html')])
        self.ch.put(data)

    def close(self):
        self.closed = True
        self.ch.put(_BREAK)

def _send_500(req, extra=None):
    req.start_response('500 Error', [('Content-Type', 'text/html')])
    req.write('
500 Internal Server Error
\n')
    req.write('The server encountered an internal error or misconfiguration and was unable to complete your request.\n')
    if extra is not None:
        req.write(extra)
    req.close()
<commit_msg>Fixed a bug in http.server<commit_after>import re
import traceback

from gevent.pywsgi import WSGIServer
from gevent.queue import Channel

from spinoff.actor import Actor
from spinoff.actor.events import Events, Error
from spinoff.actor.exceptions import Unhandled
from spinoff.util.logging import dbg
from spinoff.util.pattern_matching import ANY
from .werkzeug_wrappers import BaseRequest

class HttpServer(Actor):
    def pre_start(self, address, responders, default_content_type='text/html'):
        self.responders = responders
        self.default_content_type = default_content_type
        self.server = WSGIServer(address, self.handle_wsgi_request)
        self.server.start()

    def handle_wsgi_request(self, env, start_response):
        ch = Channel()
        req = Request(ch, env, start_response, default_content_type=self.default_content_type)
        self << ('handle', req)
        return response_stream(ch)

    def receive(self, msg):
        if ('handle', ANY) == msg:
            _, req = msg
            try:
                responder = self.get_responder(req.env['PATH_INFO'])
                if responder:
                    responder, args, kwargs = responder
                    self.spawn(RequestHandler.using(req, responder, args, kwargs))
                else:
                    req.start_response('404 Not Found', [('Content-Type', 'text/html')])
                    req.write('
404 Not Found
\n')
                    req.write('The page you tried to reach could not be found.\n')
                    req.close()
            except:
                _send_500(req)
                raise

        elif 'get-addr' == msg:
            self.reply(self.server.address)
        else:
            raise Unhandled

    def get_responder(self, path):
        for pattern, responder in self.responders:
            m = re.compile(pattern).match(path)
            if m:
                args = m.groups()
                kwargs = m.groupdict()
                return responder, args if not kwargs else (), kwargs

class RequestHandler(Actor):
    def run(self, req, responder, args, kwargs):
        try:
            self.responder = responder = self.spawn(responder.using(req, *args, **kwargs))
            self.error = None
            try:
                Events.subscribe(Error, self.check_error)  # XXX: it would be nicer if Events.subscribe accepted an actor ref to filter on
                responder.join()
                if not req.closed:
                    if self.error:
                        _send_500(req, extra='\n
\n%s
\n' % (''.join(traceback.format_exception(type(self.error.exc), self.error.exc, self.error.tb)),))
                    req.close()
            finally:
                Events.unsubscribe(Error, self.check_error)
        except:
            _send_500(req)
            raise

    def check_error(self, error):
        if error.actor == self.responder:
            self.error = error

class _BREAK(object):
    def __repr__(self):
        return '_BREAK'
_BREAK = _BREAK()

def response_stream(ch):
    while True:
        val = ch.get()
        if val is _BREAK:
            break
        yield val

class Request(BaseRequest):
    def __init__(self, ch, env, start_response, default_content_type):
        self.ch = ch
        self.env = env
        self._response_started = False
        self._start_response = start_response
        self.default_content_type = default_content_type
        self.closed = False
        BaseRequest.__init__(self, env)

    def set_status(self, status):
        self.start_response(status, [('Content-Type', self.default_content_type)])

    def start_response(self, *args):
        self._response_started = True
        ret = self._start_response(*args)
        self.start_response = self._start_response  # optimization
        return ret

    def write(self, data):
        if not self._response_started:
            self._start_response('200 OK', [('Content-Type', 'text/html')])
        self.ch.put(data)

    def close(self):
        self.closed = True
        self.ch.put(_BREAK)

def _send_500(req, extra=None):
    req.start_response('500 Error', [('Content-Type', 'text/html')])
    req.write('
500 Internal Server Error
\n')
    req.write('The server encountered an internal error or misconfiguration and was unable to complete your request.\n')
    if extra is not None:
        req.write(extra)
    req.close()
<|endoftext|>


<commit_before>import re
import traceback

from gevent.pywsgi import WSGIServer
from gevent.queue import Channel

from spinoff.actor import Actor
from spinoff.actor.events import Events, Error
from spinoff.actor.exceptions import Unhandled
from spinoff.util.pattern_matching import ANY
from werkzeug import BaseRequest

class HttpServer(Actor):
    def pre_start(self, address, responders, default_content_type='text/html'):
        self.responders = responders
        self.default_content_type = default_content_type
        self.server = WSGIServer(address, self.handle_wsgi_request)
        self.server.start()

    def post_stop(self):
        self.server.stop()

    def handle_wsgi_request(self, env, start_response):
        ch = Channel()
        req = Request(ch, env, start_response, default_content_type=self.default_content_type)
        self << ('handle', req)
        return response_stream(ch)

    def receive(self, msg):
        if ('handle', ANY) == msg:
            _, req = msg
            try:
                responder = self.get_responder(req.env['PATH_INFO'])
                if responder:
                    responder, args, kwargs = responder
                    self.spawn(RequestHandler.using(req, responder, args, kwargs))
                else:
                    req.start_response('404 Not Found', [('Content-Type', 'text/html')])
                    req.write('
404 Not Found
\n')
                    req.write('The page you tried to reach could not be found.\n')
                    req.close()
            except:
                _send_500(req)
                raise

        elif 'get-addr' == msg:
            self.reply(self.server.address)
        else:
            raise Unhandled

    def get_responder(self, path):
        for pattern, responder in self.responders:
            m = re.compile(pattern).match(path)
            if m:
                args = m.groups()
                kwargs = m.groupdict()
                return responder, args if not kwargs else (), kwargs

class RequestHandler(Actor):
    def run(self, req, responder, args, kwargs):
        try:
            self.responder = responder = self.spawn(responder.using(req, *args, **kwargs))
            self.error = None
            try:
                Events.subscribe(Error, self.check_error)  # XXX: it would be nicer if Events.subscribe accepted an actor ref to filter on
                responder.join()
                if not req.closed:
                    if self.error:
                        _send_500(req, extra='\n
\n%s
\n' % (''.join(traceback.format_exception(type(self.error.exc), self.error.exc, self.error.tb)),))
                    req.close()
            finally:
                Events.unsubscribe(Error, self.check_error)
        except:
            _send_500(req)
            raise

    def check_error(self, error):
        if error.actor == self.responder:
            self.error = error

class _BREAK(object):
    def __repr__(self):
        return '_BREAK'
_BREAK = _BREAK()

def response_stream(ch):
    while True:
        val = ch.get()
        if val is _BREAK:
            break
        yield val

class Request(BaseRequest):
    def __init__(self, ch, env, start_response, default_content_type):
        self.ch = ch
        self.env = env
        self._response_started = False
        self._start_response = start_response
        self.default_content_type = default_content_type
        self.closed = False
        BaseRequest.__init__(self, env)

    def set_status(self, status):
        self.start_response(status, [('Content-Type', self.default_content_type)])

    def start_response(self, *args):
        self._response_started = True
        ret = self._start_response(*args)
        self.start_response = self._start_response  # optimization
        return ret

    def write(self, data):
        if not self._response_started:
            self._start_response('200 OK', [('Content-Type', self.default_content_type)])
        self.ch.put(data)

    def writeln(self, data):
        self.write(data + b'\n')

    def close(self):
        if not self._response_started:
            self._start_response('200 OK', [('Content-Type', self.default_content_type)])
        self.closed = True
        self.ch.put(_BREAK)

def _send_500(req, extra=None):
    req.start_response('500 Error', [('Content-Type', 'text/html')])
    req.write('
500 Internal Server Error
\n')
    req.write('The server encountered an internal error or misconfiguration and was unable to complete your request.\n')
    if extra is not None:
        req.write(extra)
    req.close()
<commit_msg>Corrected the default_content_type parameter to contrib.http.Request to content_type<commit_after>import re
import traceback

from gevent.pywsgi import WSGIServer
from gevent.queue import Channel

from spinoff.actor import Actor
from spinoff.actor.events import Events, Error
from spinoff.actor.exceptions import Unhandled
from spinoff.util.pattern_matching import ANY
from werkzeug import BaseRequest

class HttpServer(Actor):
    def pre_start(self, address, responders, default_content_type='text/html'):
        self.responders = responders
        self.default_content_type = default_content_type
        self.server = WSGIServer(address, self.handle_wsgi_request)
        self.server.start()

    def post_stop(self):
        self.server.stop()

    def handle_wsgi_request(self, env, start_response):
        ch = Channel()
        req = Request(ch, env, start_response, default_content_type=self.default_content_type)
        self << ('handle', req)
        return response_stream(ch)

    def receive(self, msg):
        if ('handle', ANY) == msg:
            _, req = msg
            try:
                responder = self.get_responder(req.env['PATH_INFO'])
                if responder:
                    responder, args, kwargs = responder
                    self.spawn(RequestHandler.using(req, responder, args, kwargs))
                else:
                    req.start_response('404 Not Found', [('Content-Type', 'text/html')])
                    req.write('
404 Not Found
\n')
                    req.write('The page you tried to reach could not be found.\n')
                    req.close()
            except:
                _send_500(req)
                raise

        elif 'get-addr' == msg:
            self.reply(self.server.address)
        else:
            raise Unhandled

    def get_responder(self, path):
        for pattern, responder in self.responders:
            m = re.compile(pattern).match(path)
            if m:
                args = m.groups()
                kwargs = m.groupdict()
                return responder, args if not kwargs else (), kwargs

class RequestHandler(Actor):
    def run(self, req, responder, args, kwargs):
        try:
            self.responder = responder = self.spawn(responder.using(req, *args, **kwargs))
            self.error = None
            try:
                Events.subscribe(Error, self.check_error)  # XXX: it would be nicer if Events.subscribe accepted an actor ref to filter on
                responder.join()
                if not req.closed:
                    if self.error:
                        _send_500(req, extra='\n
\n%s
\n' % (''.join(traceback.format_exception(type(self.error.exc), self.error.exc, self.error.tb)),))
                    req.close()
            finally:
                Events.unsubscribe(Error, self.check_error)
        except:
            _send_500(req)
            raise

    def check_error(self, error):
        if error.actor == self.responder:
            self.error = error

class _BREAK(object):
    def __repr__(self):
        return '_BREAK'
_BREAK = _BREAK()

def response_stream(ch):
    while True:
        val = ch.get()
        if val is _BREAK:
            break
        yield val

class Request(BaseRequest):
    def __init__(self, ch, env, start_response, content_type):
        self.ch = ch
        self.env = env
        self._response_started = False
        self._start_response = start_response
        self.content_type = content_type
        self.closed = False
        BaseRequest.__init__(self, env)

    def set_status(self, status):
        self.start_response(status, [('Content-Type', self.content_type)])

    def start_response(self, *args):
        self._response_started = True
        ret = self._start_response(*args)
        self.start_response = self._start_response  # optimization
        return ret

    def write(self, data):
        if not self._response_started:
            self._start_response('200 OK', [('Content-Type', self.content_type)])
        self.ch.put(data)

    def writeln(self, data):
        self.write(data + b'\n')

    def close(self):
        if not self._response_started:
            self._start_response('200 OK', [('Content-Type', self.content_type)])
        self.closed = True
        self.ch.put(_BREAK)

def _send_500(req, extra=None):
    req.start_response('500 Error', [('Content-Type', 'text/html')])
    req.write('
500 Internal Server Error
\n')
    req.write('The server encountered an internal error or misconfiguration and was unable to complete your request.\n')
    if extra is not None:
        req.write(extra)
    req.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
API v1.0 List

'''
__author__ = 'M@Campbell'

from flask import jsonify, request, current_app, url_for
from ooiservices.app.main import api
from ooiservices.app import db
from authentication import auth
from ooiservices.app.models import Array, PlatformDeployment, InstrumentDeployment
from ooiservices.app.models import Stream, StreamParameter, Organization, Instrumentname
import json
import yaml
from wtforms import ValidationError

@api.route('/arrays')
def get_arrays():
    arrays = Array.query.all()
    return jsonify( {'arrays' : [array.to_json() for array in arrays] })

@api.route('/arrays/', methods=['POST'])
@auth.login_required
def set_array():
    array = Array.from_json(request.json)
    db.session.add(array)
    db.session.commit()
    return jsonify(array.to_json())

@api.route('/arrays/<string:id>')
def get_array(id):
    array = Array.query.filter_by(array_code=id).first_or_404()
    return jsonify(array.to_json())

@api.route('/platform_deployments')
def get_platform_deployments():
    if 'array_id' in request.args:
        platform_deployments = \
        PlatformDeployment.query.filter_by(array_id=request.args['array_id']).order_by(PlatformDeployment.reference_designator).all()
    else:
        platform_deployments = PlatformDeployment.query.all()
    return jsonify({ 'platform_deployments' : [platform_deployment.to_json() for platform_deployment in platform_deployments] })

@api.route('/platform_deployments/<string:id>')
def get_platform_deployment(id):
    platform_deployment = PlatformDeployment.query.filter_by(reference_designator=id).first_or_404()
    return jsonify(platform_deployment.to_json())

@api.route('/instrument_deployments', methods=['GET'])
def get_instrument_deployments():
    if 'platform_deployment_id' in request.args:
        instrument_deployments = \
        InstrumentDeployment.query.filter_by(platform_deployment_id=request.args['platform_deployment_id']).all()
        # TODO: Actually link the tables
        for i_d in instrument_deployments:
            instrument_name = Instrumentname.query.filter(Instrumentname.instrument_class == i_d.display_name).first()
            if instrument_name:
                i_d.display_name = instrument_name.display_name
    else:
        instrument_deployments = InstrumentDeployment.query.all()
    return jsonify({ 'instrument_deployments' : [instrument_deployment.to_json() for instrument_deployment in instrument_deployments] })

@api.route('/instrument_deployments/<string:id>', methods=['GET'])
def get_instrument_deployment(id):
    instrument_deployment = InstrumentDeployment.query.filter_by(reference_designator=id).first_or_404()
    return jsonify(instrument_deployment.to_json())

@api.route('/instrument_deployments', methods=['POST'])
@api.route('/instrument_deployments', methods=['PUT'])
def submit_deployment():
    '''
    Acts as a pass-thru proxy to to the services
    '''
    #csrf_token = session.pop('_csrf_token', None)
    #data = json.loads(request.data)
    #removes unicode
    data = yaml.load(request.data)

    # check for delete for delete
    #temp
    if 'delete' in data:
        try:
            existingDeply = InstrumentDeployment.query.filter_by(id=data['id']).first_or_404()
            db.session.delete(existingDeply)
            db.session.commit()
        except ValidationError as e:
            return jsonify(error=e.message), 409
    # check for existing id for edit
    elif 'id' in data:
        try:
            existingDeply = InstrumentDeployment.query.filter_by(id=data['id']).first_or_404()
            existingDeply.display_name = data['display_name']
            existingDeply.start_date = data['start_date']
            existingDeply.end_date = data['end_date']
            existingDeply.platform_deployment_id = data['platform_deployment_id']
            existingDeply.depth = data['depth']
            existingDeply.geo_location = data['geo_location']
            db.session.commit()
        except ValidationError as e:
            return jsonify(error=e.message), 409
    else:
        #if not csrf_token or csrf_token != data['_csrf_token']:
        #    return jsonify(error="CSRF Error"), 401
        #api_key = app.config['UI_API_KEY']
        try:
            new_deploy = InstrumentDeployment.from_json(data)
            db.session.add(new_deploy)
            db.session.commit()
        except ValidationError as e:
            return jsonify(error=e.message), 409

    return "success", 204

@api.route('/streams')
def get_streams():
    streams = Stream.query.all()
    return jsonify({ 'streams' : [stream.to_json() for stream in streams] })

@api.route('/streams/<string:id>')
def get_stream(id):
    stream = Stream.query.filter_by(stream_name=id).first_or_404()
    return jsonify(stream.to_json())

@api.route('/parameters')
def get_parameters():
    parameters = StreamParameter.query.all()
    return jsonify({ 'parameters' : [parameter.to_json() for parameter in parameters] })

@api.route('/parameters/<string:id>')
def get_parameter(id):
    parameter = StreamParameter.query.filter_by(stream_parameter_name=id).first_or_404()
    return jsonify(parameter.to_json())

@api.route('/organization', methods=['GET'])
def get_organizations():
    organizations = [o.serialize() for o in Organization.query.all()]
    return jsonify(organizations=organizations)

@api.route('/organization/<int:id>', methods=['GET'])
def get_organization_by_id(id):
    org = Organization.query.filter(Organization.id==id).first()
    if not org:
        return '{}', 204
    response = org.serialize()
    return jsonify(**response)

@api.route('/platformlocation', methods=['GET'])
def get_platform_deployment_geojson_single():
    geo_locations = {}
    if len(request.args) > 0:
        if ('reference_designator' in request.args):
            if len(request.args['reference_designator']) < 100:
                reference_designator = request.args['reference_designator']
                geo_locations = PlatformDeployment.query.filter(PlatformDeployment.reference_designator == reference_designator).all()
    else:
        geo_locations = PlatformDeployment.query.all()
    if len(geo_locations) == 0:
        return '{}', 204
    return jsonify({ 'geo_locations' : [{'id' : geo_location.id, 'reference_designator' : geo_location.reference_designator, 'geojson' : geo_location.geojson} for geo_location in geo_locations] })

@api.route('/display_name', methods=['GET'])
def get_display_name():
    # 'CE01ISSM-SBD17'
    platform_deployment_filtered = None
    display_name = ''
    if len(request.args) > 0:
        if ('reference_designator' in request.args):
            if len(request.args['reference_designator']) < 100:
                reference_designator = request.args['reference_designator']
                platform_deployment_filtered = PlatformDeployment.query.filter_by(reference_designator=reference_designator).first_or_404()
                display_name = platform_deployment_filtered.proper_display_name
    if platform_deployment_filtered is None:
        return '{}', 204
    return jsonify({ 'proper_display_name' : display_name })
<commit_msg>Makes routes more restful<commit_after>#!/usr/bin/env python
'''
API v1.0 List

'''
__author__ = 'M@Campbell'

from flask import jsonify, request, current_app, url_for
from ooiservices.app.main import api
from ooiservices.app import db
from authentication import auth
from ooiservices.app.models import Array, PlatformDeployment, InstrumentDeployment
from ooiservices.app.models import Stream, StreamParameter, Organization, Instrumentname
import json
import yaml
from wtforms import ValidationError

@api.route('/arrays')
def get_arrays():
    arrays = Array.query.all()
    return jsonify( {'arrays' : [array.to_json() for array in arrays] })

@api.route('/arrays/', methods=['POST'])
@auth.login_required
def set_array():
    array = Array.from_json(request.json)
    db.session.add(array)
    db.session.commit()
    return jsonify(array.to_json())

@api.route('/arrays/<string:id>')
def get_array(id):
    array = Array.query.filter_by(array_code=id).first_or_404()
    return jsonify(array.to_json())

@api.route('/platform_deployments')
def get_platform_deployments():
    if 'array_id' in request.args:
        platform_deployments = \
        PlatformDeployment.query.filter_by(array_id=request.args['array_id']).order_by(PlatformDeployment.reference_designator).all()
    else:
        platform_deployments = PlatformDeployment.query.all()
    return jsonify({ 'platform_deployments' : [platform_deployment.to_json() for platform_deployment in platform_deployments] })

@api.route('/platform_deployments/<string:id>')
def get_platform_deployment(id):
    platform_deployment = PlatformDeployment.query.filter_by(reference_designator=id).first_or_404()
    return jsonify(platform_deployment.to_json())

@api.route('/instrument_deployment', methods=['GET'])
def get_instrument_deployments():
    if 'platform_deployment_id' in request.args:
        instrument_deployments = \
        InstrumentDeployment.query.filter_by(platform_deployment_id=request.args['platform_deployment_id']).all()
        # TODO: Actually link the tables
        for i_d in instrument_deployments:
            instrument_name = Instrumentname.query.filter(Instrumentname.instrument_class == i_d.display_name).first()
            if instrument_name:
                i_d.display_name = instrument_name.display_name
    else:
        instrument_deployments = InstrumentDeployment.query.all()
    return jsonify({ 'instrument_deployments' : [instrument_deployment.to_json() for instrument_deployment in instrument_deployments] })

@api.route('/instrument_deployment/<int:id>', methods=['GET'])
def get_instrument_deployment(id):
    instrument_deployment = InstrumentDeployment.query.get(id)
    if instrument_deployment is None:
        return jsonify(error="Instrument Deployment Not Found"), 404
    return jsonify(**instrument_deployment.to_json())

@api.route('/instrument_deployment', methods=['POST'])
def post_instrument_deployment():
    try:
        new_deploy = InstrumentDeployment.from_json(data)
        db.session.add(new_deploy)
        db.session.commit()
    except ValidationError as e:
        return jsonify(error=e.message), 400
    return jsonify(**new_deploy.to_json()), 201

@api.route('/instrument_deployment/<int:id>', methods=['PUT'])
def put_instrument_deployment(id):
    try:
        data = json.loads(request.data)
        existingDeply = InstrumentDeployment.query.get(id)
        if existingDeply is None:
            return jsonify(error="Invalid ID, record not found"), 404
        existingDeply.display_name = data.get('display_name', existingDeply.display_name)
        existingDeply.start_date = data.get('start_date', existingDeply.start_date)
        existingDeply.end_date = data.get('end_date', existingDeply.end_date)
        existingDeply.platform_deployment_id = data.get('platform_deployment_id', existingDeply.platform_deployment_id)
        existingDeply.depth = data.get('depth', existingDeply.depth)
        existingDeply.geo_location = data.get('geo_location', existingDeply.geo_location)
        db.session.add(existingDeply)
        db.session.commit()
    except ValidationError as e:
        return jsonify(error=e.message), 400
    return jsonify(**existingDeply.to_json()), 200

@api.route('/instrument_deployment/<int:id>', methods=['DELETE'])
def delete_instrument_deployment(id):
    deployment = InstrumentDeployment.query.get(id)
    if deployment is None:
        return jsonify(error="Instrument Deployment Not Found"), 404
    db.session.delete(deployment)
    db.session.commit()
    return jsonify(), 200

#@api.route('/instrument_deployment', methods=['POST'])
#@api.route('/instrument_deployment', methods=['PUT'])
#def submit_deployment():
#    '''
#    Acts as a pass-thru proxy to to the services
#    '''
#    #csrf_token = session.pop('_csrf_token', None)
#    #data = json.loads(request.data)
#    #removes unicode
#    data = yaml.load(request.data)
#
#    # check for delete for delete
#    #temp
#    if 'delete' in data:
#        try:
#            existingDeply = InstrumentDeployment.query.filter_by(id=data['id']).first_or_404()
#            db.session.delete(existingDeply)
#            db.session.commit()
#        except ValidationError as e:
#            return jsonify(error=e.message), 409
#    # check for existing id for edit
#    elif 'id' in data:
#        try:
#        except ValidationError as e:
#            return jsonify(error=e.message), 409
#    else:
#        #if not csrf_token or csrf_token != data['_csrf_token']:
#        #    return jsonify(error="CSRF Error"), 401
#        #api_key = app.config['UI_API_KEY']
#        try:
#        except ValidationError as e:
#            return jsonify(error=e.message), 409
#
#    return "success", 204

@api.route('/streams')
def get_streams():
    streams = Stream.query.all()
    return jsonify({ 'streams' : [stream.to_json() for stream in streams] })

@api.route('/streams/<string:id>')
def get_stream(id):
    stream = Stream.query.filter_by(stream_name=id).first_or_404()
    return jsonify(stream.to_json())

@api.route('/parameters')
def get_parameters():
    parameters = StreamParameter.query.all()
    return jsonify({ 'parameters' : [parameter.to_json() for parameter in parameters] })

@api.route('/parameters/<string:id>')
def get_parameter(id):
    parameter = StreamParameter.query.filter_by(stream_parameter_name=id).first_or_404()
    return jsonify(parameter.to_json())

@api.route('/organization', methods=['GET'])
def get_organizations():
    organizations = [o.serialize() for o in Organization.query.all()]
    return jsonify(organizations=organizations)

@api.route('/organization/<int:id>', methods=['GET'])
def get_organization_by_id(id):
    org = Organization.query.filter(Organization.id==id).first()
    if not org:
        return '{}', 204
    response = org.serialize()
    return jsonify(**response)

@api.route('/platformlocation', methods=['GET'])
def get_platform_deployment_geojson_single():
    geo_locations = {}
    if len(request.args) > 0:
        if ('reference_designator' in request.args):
            if len(request.args['reference_designator']) < 100:
                reference_designator = request.args['reference_designator']
                geo_locations = PlatformDeployment.query.filter(PlatformDeployment.reference_designator == reference_designator).all()
    else:
        geo_locations = PlatformDeployment.query.all()
    if len(geo_locations) == 0:
        return '{}', 204
    return jsonify({ 'geo_locations' : [{'id' : geo_location.id, 'reference_designator' : geo_location.reference_designator, 'geojson' : geo_location.geojson} for geo_location in geo_locations] })

@api.route('/display_name', methods=['GET'])
def get_display_name():
    # 'CE01ISSM-SBD17'
    platform_deployment_filtered = None
    display_name = ''
    if len(request.args) > 0:
        if ('reference_designator' in request.args):
            if len(request.args['reference_designator']) < 100:
                reference_designator = request.args['reference_designator']
                platform_deployment_filtered = PlatformDeployment.query.filter_by(reference_designator=reference_designator).first_or_404()
                display_name = platform_deployment_filtered.proper_display_name
    if platform_deployment_filtered is None:
        return '{}', 204
    return jsonify({ 'proper_display_name' : display_name })
<|endoftext|>


<commit_before>#!/usr/bin/python

"""
Copyright 2016 EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# -*- coding: utf-8 -*-
'''process whole genome bisulfate sequencing FastQ files'''
import argparse, time, urllib2, gzip, shutil
from pycompss.api.task import task
from pycompss.api.parameter import *
from pycompss.api.constraint import *

from common import common

from fastqreader import *
from FilterReads import *
from bs_index.wg_build import *

class process_wgbs:
    """
    Functions for downloading and processing whole genome bisulfate sequencings
    (WGBS) files. Files are downloaded from the European Nucleotide Archive
    (ENA), then filtered, aligned and analysed for points of methylation
    """

    def __init__ (self):
        """
        Initialise the module
        """
        self.ready=None

    #@task(infile = FILE_IN, outfile = FILE_OUT, returns = int)
    def FilterFastQReads(self, infile, outfile):
        """
        This is optional, but removes reads that can be problematic for the
        alignment of whole genome datasets.

        If performing RRBS then this step can be skipped
        """
        FilterReads(infile, outfile, True)
        return 1

    #@constraint(ProcessorCoreCount=8)
    #@task(fasta_file = FILE_IN, aligner = IN, aligner_path = IN, ref_path = IN)
    def Builder(self, fasta_file, aligner, aligner_path, ref_path):
        """
        Function to submit the FASTA file for the reference sequence and build
        the required index file used by the aligner.

        This only needs to be done once, so there needs to be a check to ensure
        that if the index file have already been generated then they do no need
        to be analysed again
        """
        builder_exec = os.path.join(aligner_path,
                                    {BOWTIE   : 'bowtie-build',
                                     BOWTIE2  : 'bowtie2-build',
                                     SOAP     : '2bwt-builder',
                                     RMAP     : '' # do nothing
                                    }[aligner])

        build_command = builder_exec + { BOWTIE   : ' -f %(fname)s.fa %(fname)s',
                                         BOWTIE2  : ' -f %(fname)s.fa %(fname)s',
                                         SOAP     : ' %(fname)s.fa'
                                       }[aligner]

        wg_build(fasta_file, build_command, ref_path, aligner)

    def Splitter(self, in_file1, in_file2, tag):
        """
        Function to divide the FastQ files into separte sub files of 1000000
        sequences so that the aligner can get run in parallel.

        Returns: Returns a list of lists of the files that have been generated.
                 Each sub list containing the two paired end files for that
                 subset.
        """

        fqr = fastqreader()
        fqr.openPairedFastQ(in_file1, in_file2)
        fqr.createOutputFiles(tag)

        r1 = fqr.next(1)
        r2 = fqr.next(2)

        count_r1 = 0
        count_r2 = 0
        count_r3 = 0
        files_out = []

        while fqr.eof(1) == False and fqr.eof(2) == False:
            r1_id = r1["id"].split(" ")
            r2_id = r2["id"].split(" ")

            if r1_id[0] == r2_id[0]:
                fqr.writeOutput(r1, 1)
                fqr.writeOutput(r2, 2)

                r1 = fqr.next(1)
                r2 = fqr.next(2)

                count_r1 += 1
                count_r2 += 1
                count_r3 += 1
            elif r1_id[0] < r2_id[0]:
                r1 = fqr.next(1)
                count_r1 += 1
            else:
                r2 = fqr.next(2)
                count_r2 += 1

            if count_r3 % 1000000 == 0:
                fqr.incrementOutputFiles()
                f1 = fqr.fastq1.split("/")
                f1[-1] = f1[-1].replace(".fastq", "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
                f1.insert(-1, "tmp")

                f2 = fqr.fastq2.split("/")
                f2[-1] = f2[-1].replace(".fastq", "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
                f2.insert(-1, "tmp")

                files_out.append(["/".join(f1), "/".join(f2)])

        fqr.closePairedFastQ()
        fqr.closeOutputFiles()

        return files_out

    #@constraint(ProcessorCoreCount=8)
    #@task(input_fastq1 = FILE_IN, input_fastq2 = FILE_IN, aligner = IN, aligner_path = IN, genome_fasta = FILE_IN, returns = int)
    def Aligner(self, input_fastq1, input_fastq2, aligner, aligner_path, genome_fasta):
        """
        Alignment of the paired ends to the reference genome
        Generates bam files for the alignments
        This is performed by running the external program rather than reimplementing
        the code from the main function to make it easier when it comes to updating
        the changes in BS-Seeker2
        """
        import subprocess

        args = ["python", "bs_seeker2-align.py", "--input_1", input_fastq1, "--input_2", input_fastq2, "--aligner", aligner, "--path", aligner_path_path, "--genome", genome_fasta, "--bt2-p", "4"]
        p = subprocess.Popen(args)
        p.wait()

        return 1

    #@constraint(ProcessorCoreCount=8)
    #@task(bam_file = FILE_IN, output_prefix = IN, db_dir = IN, returns = int)
    def MethylationCaller(self, bam_file, output_prefix, db_dir):
        """
        Takes the merged and sorted bam file and calls the methylation sites.
        Generates a wig file of the potential sites.
        This is performed by running the external program rather than reimplementing
        the code from the main function to make it easier when it comes to updating
        the changes in BS-Seeker2
        """
        import subprocess

        args = ["python", "bs_seeker2-call_methylation.py", "--input", str(bam_file), "--output-prefix", str(output_prefix), "--db", db_dir]
        p = subprocess.Popen(args)
        p.wait()

        return 1

    def clean_up(self, data_dir):
        """
        Clears up the tmp folders
        """
        os.chdir(self.data_dir)

        try:
            shutil.rmtree("tmp")
        except:
            pass

if __name__ == "__main__":
    import sys
    import os

    # Set up the command line parameters
    parser = argparse.ArgumentParser(description="Parse WGBS data")
    parser.add_argument("--species", help="Species (Homo_sapiens)")
    parser.add_argument("--assembly", help="Assembly (GRCm38)")
    #parser.add_argument("--genome", help="Genome name") #             default="GCA_000001405.22")
    parser.add_argument("--project_id", help="Project ID of the dataset") #   default="PRJNA257496")
    parser.add_argument("--srr_id", help="SRR ID of the dataset") #   default="SRR1536575")
    parser.add_argument("--aligner", help="Aligner to use (eg bowtie2)") #   default="bowtie2")
    parser.add_argument("--tmp_dir", help="Temporary data dir")
    parser.add_argument("--data_dir", help="Data directory; location to download SRA FASTQ files and save results")
    parser.add_argument("--aligner_dir", help="Directory for the aligner program")
    parser.add_argument("--local", help="Directory and data files already available", default=0)

    # Get the matching parameters from the command line
    args = parser.parse_args()

    project_id = args.project_id
    srr_id   = args.srr_id
    #genome   = args.genome
    species     = args.species
    assembly    = args.assembly
    aligner  = args.aligner
    aligner_dir = args.aligner_dir
    data_dir = args.data_dir
    tmp_dir  = args.tmp_dir
    local = args.local

    start = time.time()

    db_dir = ""

    pwgbs = process_wgbs()

    if data_dir[-1] != "/":
        data_dir += "/"

    try:
        os.makedirs(data_dir)
    except:
        pass

    try:
        os.makedirs(data_dir + project_id + '/' + srr_id)
    except:
        pass

    try:
        os.makedirs(data_dir + project_id + '/tmp')
    except:
        pass

    genome_dir = args.data_dir + species + "_" + assembly

    try:
        os.makedirs(genome_dir)
    except:
        pass

    cf = common()

    # Optain the paired FastQ files
    if (local == 0):
        in_files = cf.getFastqFiles(project_id, data_dir, srr_id)
    else:
        in_files = [f for f in os.listdir(data_dir + project_id + '/' + srr_id) if re.match(srr_id, f)]

    in_file1 = in_files[0]
    in_file2 = in_files[1]
    out_file1 = in_file1.replace(".fastq", "_filtered.fastq")
    out_file2 = in_file2.replace(".fastq", "_filtered.fastq")

    # Get the assembly
    genome_fa = cf.getGenomeFromENA(data_dir, species, assembly, False)

    # Run the FilterReads.py steps for the individual FastQ files
    for l in [[in_file1, out_file1], [in_file2, out_file2]]:
        pwgbs.FilterFastQReads(l[0], l[1])

    # Run the bs_seeker2-builder.py steps
    pwgbs.Builder(genome_fa["unzipped"], "bowtie2", aligner_dir, genome_dir)

    # Split the paired fastq files
    tmp_fastq = pwgbs.Splitter(in_file1, in_file2, 'tmp')

    bam_sort_files = []
    bam_merge_files = []
    fastq_for_alignment = []
    for bams in tmp_fastq:
        tmp = bams
        tmp.append(aligner)
        tmp.append(aligner_path)
        tmp.append(genome_fa["unzipped"])
        fastq_for_alignment.append(tmp)
        bam_root = bams[0] + "_bspe.bam"
        bam_sort_files.append(bams[0] + "_bspe.bam")
        bam_merge_files.append(["-o", bam_root + ".sorted.bam", "-T", bam_root + ".bam_sort", bam_root])

    # Run the bs_seeker2-align.py steps on the split up fastq files
    for ffa in fastq_for_alignment:
        pwgbs.Aligner(ffa)

    # Sort and merge the aligned bam files
    # Pre-sort the original input bam files
    x = map(pysam.sort, bam_sort_files)

    f_bam = in_file1.split("/")
    f_bam[-1] = f_bam[-1].replace(".fastq", ".sorted.bam")
    out_bam_file = "/".join(f_bam)

    pysam.merge(out_bam_file, *bam_merge_files)

    pysam.sort("-o", out_bam_file + '.sorted.bam', "-T", out_bam_file + ".bam_sort", out_bam_file)

    pysam.index(out_bam_file)

    # Run the bs_seeker2-call_methylation.py steps
    x = pwgbs.MethylationCaller(out_bam_file, 'tmp', db_dir)
    x = compss_wait_on(x)

    # Tidy up
    pwgbs.clean_up(data_dir + srr_id)

<commit_msg>Fixes to the code so that it runs in the VM correctly<commit_after>#!/usr/bin/python

"""
Copyright 2016 EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# -*- coding: utf-8 -*-
'''process whole genome bisulfate sequencing FastQ files'''
import argparse, time, urllib2, gzip, shutil
from pycompss.api.task import task
from pycompss.api.parameter import *
from pycompss.api.constraint import *

from common import common
import pysam

from fastqreader import *
from FilterReads import *
from bs_index.wg_build import *

class process_wgbs:
    """
    Functions for downloading and processing whole genome bisulfate sequencings
    (WGBS) files. Files are downloaded from the European Nucleotide Archive
    (ENA), then filtered, aligned and analysed for points of methylation
    """

    def __init__ (self):
        """
        Initialise the module
        """
        self.ready=None

    #@task(infile = FILE_IN, outfile = FILE_OUT, returns = int)
    def FilterFastQReads(self, infile, outfile):
        """
        This is optional, but removes reads that can be problematic for the
        alignment of whole genome datasets.

        If performing RRBS then this step can be skipped
        """
        FilterReads(infile, outfile, True)
        return 1

    #@constraint(ProcessorCoreCount=8)
    #@task(fasta_file = FILE_IN, aligner = IN, aligner_path = IN, ref_path = IN)
    def Builder(self, fasta_file, aligner, aligner_path, ref_path):
        """
        Function to submit the FASTA file for the reference sequence and build
        the required index file used by the aligner.

        This only needs to be done once, so there needs to be a check to ensure
        that if the index file have already been generated then they do no need
        to be analysed again
        """
        builder_exec = os.path.join(aligner_path,
                                    {BOWTIE   : 'bowtie-build',
                                     BOWTIE2  : 'bowtie2-build',
                                     SOAP     : '2bwt-builder',
                                     RMAP     : '' # do nothing
                                    }[aligner])

        build_command = builder_exec + { BOWTIE   : ' -f %(fname)s.fa %(fname)s',
                                         BOWTIE2  : ' -f %(fname)s.fa %(fname)s',
                                         SOAP     : ' %(fname)s.fa'
                                       }[aligner]

        wg_build(fasta_file, build_command, ref_path, aligner)

    def Splitter(self, in_file1, in_file2, tag):
        """
        Function to divide the FastQ files into separte sub files of 1000000
        sequences so that the aligner can get run in parallel.

        Returns: Returns a list of lists of the files that have been generated.
                 Each sub list containing the two paired end files for that
                 subset.
        """

        fqr = fastqreader()
        fqr.openPairedFastQ(in_file1, in_file2)
        fqr.createOutputFiles(tag)

        r1 = fqr.next(1)
        r2 = fqr.next(2)

        count_r1 = 0
        count_r2 = 0
        count_r3 = 0

        f1 = fqr.fastq1.split("/")
        f1[-1] = f1[-1].replace(".fastq", "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
        f1.insert(-1, "tmp")

        f2 = fqr.fastq2.split("/")
        f2[-1] = f2[-1].replace(".fastq", "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
        f2.insert(-1, "tmp")
        files_out = [["/".join(f1), "/".join(f2)]]

        while fqr.eof(1) == False and fqr.eof(2) == False:
            r1_id = r1["id"].split(" ")
            r2_id = r2["id"].split(" ")

            if r1_id[0] == r2_id[0]:
                fqr.writeOutput(r1, 1)
                fqr.writeOutput(r2, 2)

                r1 = fqr.next(1)
                r2 = fqr.next(2)

                count_r1 += 1
                count_r2 += 1
                count_r3 += 1
            elif r1_id[0] < r2_id[0]:
                r1 = fqr.next(1)
                count_r1 += 1
            else:
                r2 = fqr.next(2)
                count_r2 += 1

            if count_r3 % 1000000 == 0:
                fqr.incrementOutputFiles()
                f1 = fqr.fastq1.split("/")
                f1[-1] = f1[-1].replace(".fastq", "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
                f1.insert(-1, "tmp")

                f2 = fqr.fastq2.split("/")
                f2[-1] = f2[-1].replace(".fastq", "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
                f2.insert(-1, "tmp")

                files_out.append(["/".join(f1), "/".join(f2)])

        fqr.closePairedFastQ()
        fqr.closeOutputFiles()

        return files_out

    #@constraint(ProcessorCoreCount=8)
    #@task(input_fastq1 = FILE_IN, input_fastq2 = FILE_IN, aligner = IN, aligner_path = IN, genome_fasta = FILE_IN, returns = int)
    def Aligner(self, input_fastq1, input_fastq2, aligner, aligner_path, genome_fasta, bam_out):
        """
        Alignment of the paired ends to the reference genome
        Generates bam files for the alignments
        This is performed by running the external program rather than reimplementing
        the code from the main function to make it easier when it comes to updating
        the changes in BS-Seeker2
        """
        import subprocess

        g_dir = genome_fasta.split("/")
        g_dir = "/".join(g_dir[0:-1])

        command_line = "python " + aligner_path + "/bs_seeker2-align.py --input_1 " + input_fastq1 + " --input_2 " + input_fastq2 + " --aligner " + aligner + " --path " + aligner_path + " --genome " + genome_fasta + " -d " + g_dir + " --bt2-p 4 -o " + bam_out

        args = shlex.split(command_line)
        p = subprocess.Popen(args)
        p.wait()

        return 1

    #@constraint(ProcessorCoreCount=8)
    #@task(bam_file = FILE_IN, output_prefix = IN, db_dir = IN, returns = int)
    def MethylationCaller(self, aligner_path, bam_file, output_prefix, db_dir):
        """
        Takes the merged and sorted bam file and calls the methylation sites.
        Generates a wig file of the potential sites.
        This is performed by running the external program rather than reimplementing
        the code from the main function to make it easier when it comes to updating
        the changes in BS-Seeker2
        """
        import subprocess

        command_line = "python " + aligner_path + "/bs_seeker2-call_methylation.py --sorted --input " + str(bam_file) + " --output-prefix " + str(output_prefix) + " --db " + db_dir
        args = shlex.split(command_line)
        p = subprocess.Popen(args)
        p.wait()

        return 1

    def clean_up(self, data_dir):
        """
        Clears up the tmp folders
        """
        os.chdir(self.data_dir)

        try:
            shutil.rmtree("tmp")
        except:
            pass

if __name__ == "__main__":
    import sys
    import os

    # Set up the command line parameters
    parser = argparse.ArgumentParser(description="Parse WGBS data")
    parser.add_argument("--species", help="Species (Homo_sapiens)")
    parser.add_argument("--assembly", help="Assembly (GRCm38)")
    #parser.add_argument("--genome", help="Genome name") #             default="GCA_000001405.22")
    parser.add_argument("--project_id", help="Project ID of the dataset") #   default="PRJNA257496")
    parser.add_argument("--srr_id", help="SRR ID of the dataset") #   default="SRR1536575")
    parser.add_argument("--aligner", help="Aligner to use (eg bowtie2)") #   default="bowtie2")
    parser.add_argument("--tmp_dir", help="Temporary data dir")
    parser.add_argument("--data_dir", help="Data directory; location to download SRA FASTQ files and save results")
    parser.add_argument("--aligner_dir", help="Directory for the aligner program")
    parser.add_argument("--local", help="Directory and data files already available", default=0)

    # Get the matching parameters from the command line
    args = parser.parse_args()

    project_id = args.project_id
    srr_id   = args.srr_id
    #genome   = args.genome
    species     = args.species
    assembly    = args.assembly
    aligner  = args.aligner
    aligner_dir = args.aligner_dir
    data_dir = args.data_dir
    tmp_dir  = args.tmp_dir
    local = args.local

    start = time.time()

    db_dir = ""

    pwgbs = process_wgbs()

    if data_dir[-1] != "/":
        data_dir += "/"

    try:
        os.makedirs(data_dir)
    except:
        pass

    try:
        os.makedirs(data_dir + project_id + '/' + srr_id)
    except:
        pass

    try:
        os.makedirs(data_dir + project_id + '/tmp')
    except:
        pass

    genome_dir = data_dir + species + "_" + assembly

    try:
        os.makedirs(genome_dir)
    except:
        pass

    cf = common()

    # Optain the paired FastQ files
    if (local == 0):
        in_files = cf.getFastqFiles(project_id, data_dir, srr_id)
    else:
        in_files = [f for f in os.listdir(data_dir + project_id + '/' + srr_id) if re.match(srr_id, f)]

    in_file1 = in_files[0]
    in_file2 = in_files[1]
    out_file1 = in_file1.replace(".fastq", "_filtered.fastq")
    out_file2 = in_file2.replace(".fastq", "_filtered.fastq")

    # Get the assembly
    genome_fa = cf.getGenomeFromENA(data_dir, species, assembly, False)

    # Run the FilterReads.py steps for the individual FastQ files
    for l in [[in_file1, out_file1], [in_file2, out_file2]]:
        pwgbs.FilterFastQReads(l[0], l[1])

    # Run the bs_seeker2-builder.py steps
    pwgbs.Builder(genome_fa["unzipped"], "bowtie2", aligner_dir, genome_dir)

    # Split the paired fastq files
    tmp_fastq = pwgbs.Splitter(in_file1, in_file2, 'tmp')
    bam_sort_files = []
    bam_merge_files = []
    fastq_for_alignment = []
    for bams in tmp_fastq:
        tmp = bams
        tmp.append(aligner)
        tmp.append(aligner_path)
        tmp.append(genome_fa["unzipped"])
        tmp.append(bams[0] + "_bspe.bam")
        fastq_for_alignment.append(tmp)
        bam_root = bams[0] + "_bspe.bam"
        bam_sort_files.append([bam_root, bam_root + ".sorted.bam"])
        bam_merge_files.append(bam_root + ".sorted.bam")

    # Run the bs_seeker2-align.py steps on the split up fastq files
    for ffa in fastq_for_alignment:
        pwgbs.Aligner(ffa[0], ffa[1], ffa[2], ffa[3][ffa[2]], ffa[4], ffa[5])

    # Sort and merge the aligned bam files
    # Pre-sort the original input bam files
    for bfs in bam_sort_files:
        pysam.sort("-o", bfs[1], bfs[0])

    f_bam = in_file1.split("/")
    f_bam[-1] = f_bam[-1].replace(".fastq", ".sorted.bam")
    out_bam_file = "/".join(f_bam)

    pysam.merge(out_bam_file, *bam_merge_files)

    pysam.sort("-o", out_bam_file + '.sorted.bam', "-T", out_bam_file + ".bam_sort", out_bam_file)

    pysam.index(out_bam_file)

    # Run the bs_seeker2-call_methylation.py steps
    pwgbs.MethylationCaller(aligner_dir, out_bam_file, data_dir + project_id + '/' + srr_id + '/' + srr_id, genome_fa["unzipped"] + "_bowtie2")

    # Tidy up
    pwgbs.clean_up(data_dir + srr_id)

<|endoftext|>


<commit_before><commit_msg>Minor fix to setup command<commit_after><|endoftext|>


<commit_before><commit_msg>Fixing paginating recommendation part 3<commit_after><|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
import scipy.misc

def down_sampling(img):
    shape = img.get_shape()  # [batch, height, width, channels]

    h2 = int(shape[1] // 2)
    w2 = int(shape[2] // 2)

    return tf.image.resize_images(img, [h2, w2], tf.image.ResizeMethod.BILINEAR)

def up_sampling(img):
    shape = img.get_shape()  # [batch, height, width, channels]

    h2 = int(shape[1] * 2)
    w2 = int(shape[2] * 2)

    return tf.image.resize_images(img, [h2, w2], tf.image.ResizeMethod.BILINEAR)

def inverse_transform(images, inv_type='255'):
    if inv_type == '255':    # [ 0  1]
        images *= 255
    elif inv_type == '127':  # [-1, 1]
        images = (images + 1) * (255 / 2.)

    # clipped by [0, 255]
    images[images > 255] = 255
    images[images < 0] = 0

    return images.astype(np.uint8)

def merge(images, size):
    h, w = images.shape[1], images.shape[2]

    img = np.zeros((h * size[0], w * size[1], 3))
    for idx, image in enumerate(images):
        i = idx % size[1]
        j = idx // size[1]
        img[j * h:j * h + h, i * w:i * w + w, :] = image

    return img

def save_image(images, size, path):
    return scipy.misc.imsave(path, merge(images, size))

def save_images(images, size, image_path, inv_type='255'):
    return save_image(inverse_transform(images, inv_type), size, image_path)

def img_save(img, path, inv_type='255'):
    return scipy.misc.imsave(path, img)
    # return cv2.imwrite(path, inverse_transform(img, inv_type))
<commit_msg>move: image sampling funcs to tfutil.py<commit_after>import numpy as np
import scipy.misc

def inverse_transform(images, inv_type='255'):
    if inv_type == '255':    # [ 0  1]
        images *= 255
    elif inv_type == '127':  # [-1, 1]
        images = (images + 1) * (255 / 2.)

    # clipped by [0, 255]
    images[images > 255] = 255
    images[images < 0] = 0

    return images.astype(np.uint8)

def merge(images, size):
    h, w = images.shape[1], images.shape[2]

    img = np.zeros((h * size[0], w * size[1], 3))
    for idx, image in enumerate(images):
        i = idx % size[1]
        j = idx // size[1]
        img[j * h:j * h + h, i * w:i * w + w, :] = image

    return img

def save_image(images, size, path):
    return scipy.misc.imsave(path, merge(images, size))

def save_images(images, size, image_path, inv_type='255'):
    return save_image(inverse_transform(images, inv_type), size, image_path)

def img_save(img, path, inv_type='255'):
    return scipy.misc.imsave(path, img)
    # return cv2.imwrite(path, inverse_transform(img, inv_type))
<|endoftext|>


<commit_before><commit_msg>[crdata] removed comments<commit_after><|endoftext|>