<commit_before>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import os

from pypeline.node import CommandNode
from pypeline.atomiccmd import AtomicCmd
from pypeline.atomicparams import *
from pypeline.atomicset import ParallelCmds

class BWAIndexNode(CommandNode):
    @create_customizable_cli_parameters
    def customize(cls, input_file, prefix = None, dependencies = ()):
        params = AtomicParams("bwa")
        params.set_parameter("index")
        params.set_parameter("%(IN_FILE)s")

        # Destination prefix, in temp folder
        params.set_parameter("-p", "%(TEMP_OUT_PREFIX)s")

        prefix = prefix if prefix else input_file
        params.set_paths(IN_FILE = input_file,
                         TEMP_OUT_PREFIX = os.path.basename(prefix),
                         **_prefix_files(prefix, iotype = "OUT"))

        return {"prefix":  prefix,
                "command": params}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        command = parameters.command.finalize()
        description =  "<BWA Index '%s' -> '%s.*'>" % (parameters.input_file,
                                                       parameters.prefix)
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             dependencies = parameters.dependencies)

class SE_BWANode(CommandNode):
    @create_customizable_cli_parameters
    def customize(self, input_file, output_file, reference, prefix, threads = 1, dependencies = ()):
        aln   = AtomicParams("bwa")
        aln.set_parameter("aln")
        aln.set_parameter(prefix)
        aln.set_parameter("%(IN_FILE)s")
        aln.set_parameter("-t", threads)
        aln.set_paths(IN_FILE = input_file,
                      OUT_STDOUT = AtomicCmd.PIPE,
                      **_prefix_files(prefix))

        samse = AtomicParams("bwa")
        samse.set_parameter("samse")
        samse.set_parameter(prefix)
        samse.set_parameter("-")
        samse.set_parameter("%(IN_FILE)s")
        samse.set_paths(IN_STDIN = aln,
                        IN_FILE  = input_file,
                        OUT_STDOUT = AtomicCmd.PIPE)

        order, commands = _process_output(samse, output_file, reference)
        commands["samse"] = samse
        commands["aln"]   = aln

        return {"commands" : commands,
                "order"    : ["aln", "samse"] + order}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        command = ParallelCmds([parameters.commands[key].finalize() for key in parameters.order])
        description =  "<SE_BWA (%i threads): '%s'>" % (parameters.threads, parameters.input_file)
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             threads      = parameters.threads,
                             dependencies = parameters.dependencies)

class PE_BWANode(CommandNode):
    @create_customizable_cli_parameters
    def customize(cls, input_file_1, input_file_2, output_file, reference, prefix, threads = 1, dependencies = ()):
        alns = []
        for (iindex, filename) in enumerate((input_file_1, input_file_2), start = 1):
            aln = AtomicParams("bwa")
            aln.set_parameter("aln")
            aln.set_parameter(prefix)
            aln.set_parameter("%(IN_FILE)s")
            aln.set_parameter("-f", "%(TEMP_OUT_SAI)s")
            aln.set_parameter("-t", max(1, threads / 2))
            aln.set_paths(IN_FILE = filename,
                          OUT_STDOUT = AtomicCmd.PIPE,
                          TEMP_OUT_SAI = "pair_%i.sai" % iindex,
                          **_prefix_files(prefix))
            alns.append(aln)
        aln_1, aln_2 = alns

        sampe = AtomicParams("bwa")
        sampe.set_parameter("sampe")
        sampe.set_parameter(prefix)
        sampe.set_parameter("%(TEMP_IN_SAI_1)s")
        sampe.set_parameter("%(TEMP_IN_SAI_2)s")
        sampe.set_parameter("%(IN_FILE_1)s")
        sampe.set_parameter("%(IN_FILE_2)s")
        sampe.set_parameter("-P", fixed = False)
        sampe.set_paths(IN_FILE_1     = input_file_1,
                        IN_FILE_2     = input_file_2,
                        TEMP_IN_SAI_1 = "pair_1.sai",
                        TEMP_IN_SAI_2 = "pair_2.sai",
                        OUT_STDOUT    = AtomicCmd.PIPE)

        order, commands = _process_output(sampe, output_file, reference)
        commands["sampe"] = sampe
        commands["aln_1"] = aln_1
        commands["aln_2"] = aln_2

        return {"commands" : commands,
                "order"    : ["aln_1", "aln_2", "sampe"] + order}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        command = ParallelCmds([parameters.commands[key].finalize() for key in parameters.order])
        description =  "<PE_BWA (%i threads): '%s'>" % (parameters.threads, parameters.input_file_1)
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             threads      = parameters.threads,
                             dependencies = parameters.dependencies)

    def _setup(self, _config, temp):
        os.mkfifo(os.path.join(temp, "pair_1.sai"))
        os.mkfifo(os.path.join(temp, "pair_2.sai"))

class BWASWNode(CommandNode):
    @create_customizable_cli_parameters
    def customize(cls, input_file_1, output_file, reference, prefix, input_file_2 = None, threads = 1, dependencies = ()):
        aln = AtomicParams("bwa")
        aln.set_parameter("bwasw")
        aln.set_parameter(prefix)
        aln.set_parameter("%(IN_FILE_1)s")

        files   = {"IN_FILE_1"  : input_file_1,
                   "OUT_STDOUT" : AtomicCmd.PIPE}
        if input_file_2:
            aln.set_parameter("%(IN_FILE_2)s")
            files["IN_FILE_2"] = input_file_2
        files.update(_prefix_files(prefix))
        aln.set_paths(**files)

        aln.set_parameter("-t", threads)

        order, commands = _process_output(aln, output_file, reference)
        commands["aln"] = aln

        return {"commands" : commands,
                "order"    : ["aln"] + order}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        if parameters.input_file_2:
            description =  "<PE_BWASW (%i threads): '%s', '%s' -> '%s'>" \
                % (parameters.threads, parameters.input_file_1, parameters.input_file_2, parameters.output_file)
        else:
            description =  "<BWASW (%i threads): '%s' -> '%s'>" \
                % (parameters.threads, parameters.input_file_1, parameters.output_file)

        command = ParallelCmds([parameters.commands[key].finalize() for key in parameters.order])
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             threads      = parameters.threads,
                             dependencies = parameters.dependencies)

def _process_output(stdin, output_file, reference):
    convert = AtomicParams("safeSAM2BAM")
    convert.set_parameter("--flag-as-sorted")
    convert.set_paths(IN_STDIN   = stdin,
                      OUT_STDOUT = AtomicCmd.PIPE)

    flt = AtomicParams("samtools")
    flt.set_parameter("view")
    flt.set_parameter("-")
    flt.set_parameter("-b") # Output BAM
    flt.set_parameter("-u") # Output uncompressed BAM
    flt.set_parameter("-F", "0x4", sep = "", fixed = False) # Remove misses
    flt.set_paths(IN_STDIN  = convert,
                  OUT_STDOUT = AtomicCmd.PIPE)

    sort = AtomicParams("samtools")
    sort.set_parameter("sort")
    sort.set_parameter("-o") # Output to STDOUT on completion
    sort.set_parameter("-")
    sort.set_parameter("%(TEMP_OUT_BAM)s")
    sort.set_paths(IN_STDIN     = flt,
                   OUT_STDOUT   = AtomicCmd.PIPE,
                   TEMP_OUT_BAM = "sorted")

    calmd = AtomicParams("samtools")
    calmd.set_parameter("calmd")
    calmd.set_parameter("-b") # Output BAM
    calmd.set_parameter("-")
    calmd.set_parameter("%(IN_REF)s")
    calmd.set_paths(IN_REF   = reference,
                    IN_STDIN = sort,
                    OUT_STDOUT = output_file)

    order = ["convert", "filter", "sort", "calmd"]
    dd = {"convert" : convert,
          "filter"  : flt,
          "sort"    : sort,
          "calmd"   : calmd}

    return order, dd

def _prefix_files(prefix, iotype = "IN"):
    files = {}
    for postfix in ("amb", "ann", "bwt", "pac", "rbwt", "rpac", "rsa", "sa"):
        files["%s_PREFIX_%s" % (iotype, postfix.upper())] = prefix + "." + postfix
    return files
<commit_msg>Limit BWA thread usage for very small genomes.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import os

from pypeline.node import CommandNode
from pypeline.atomiccmd import AtomicCmd
from pypeline.atomicparams import *
from pypeline.atomicset import ParallelCmds

class BWAIndexNode(CommandNode):
    @create_customizable_cli_parameters
    def customize(cls, input_file, prefix = None, dependencies = ()):
        params = AtomicParams("bwa")
        params.set_parameter("index")
        params.set_parameter("%(IN_FILE)s")

        # Destination prefix, in temp folder
        params.set_parameter("-p", "%(TEMP_OUT_PREFIX)s")

        prefix = prefix if prefix else input_file
        params.set_paths(IN_FILE = input_file,
                         TEMP_OUT_PREFIX = os.path.basename(prefix),
                         **_prefix_files(prefix, iotype = "OUT"))

        return {"prefix":  prefix,
                "command": params}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        command = parameters.command.finalize()
        description =  "<BWA Index '%s' -> '%s.*'>" % (parameters.input_file,
                                                       parameters.prefix)
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             dependencies = parameters.dependencies)

class SE_BWANode(CommandNode):
    @create_customizable_cli_parameters
    def customize(self, input_file, output_file, reference, prefix, threads = 1, dependencies = ()):
        threads = _get_max_threads(reference, threads)

        aln   = AtomicParams("bwa")
        aln.set_parameter("aln")
        aln.set_parameter(prefix)
        aln.set_parameter("%(IN_FILE)s")
        aln.set_parameter("-t", threads)
        aln.set_paths(IN_FILE = input_file,
                      OUT_STDOUT = AtomicCmd.PIPE,
                      **_prefix_files(prefix))

        samse = AtomicParams("bwa")
        samse.set_parameter("samse")
        samse.set_parameter(prefix)
        samse.set_parameter("-")
        samse.set_parameter("%(IN_FILE)s")
        samse.set_paths(IN_STDIN = aln,
                        IN_FILE  = input_file,
                        OUT_STDOUT = AtomicCmd.PIPE)

        order, commands = _process_output(samse, output_file, reference)
        commands["samse"] = samse
        commands["aln"]   = aln

        return {"commands" : commands,
                "order"    : ["aln", "samse"] + order,
                "threads"  : threads}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        command = ParallelCmds([parameters.commands[key].finalize() for key in parameters.order])
        description =  "<SE_BWA (%i threads): '%s'>" % (parameters.threads, parameters.input_file)
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             threads      = parameters.threads,
                             dependencies = parameters.dependencies)

class PE_BWANode(CommandNode):
    @create_customizable_cli_parameters
    def customize(cls, input_file_1, input_file_2, output_file, reference, prefix, threads = 1, dependencies = ()):
        threads = _get_max_threads(reference, threads)

        alns = []
        for (iindex, filename) in enumerate((input_file_1, input_file_2), start = 1):
            aln = AtomicParams("bwa")
            aln.set_parameter("aln")
            aln.set_parameter(prefix)
            aln.set_parameter("%(IN_FILE)s")
            aln.set_parameter("-f", "%(TEMP_OUT_SAI)s")
            aln.set_parameter("-t", max(1, threads / 2))
            aln.set_paths(IN_FILE = filename,
                          OUT_STDOUT = AtomicCmd.PIPE,
                          TEMP_OUT_SAI = "pair_%i.sai" % iindex,
                          **_prefix_files(prefix))
            alns.append(aln)
        aln_1, aln_2 = alns

        sampe = AtomicParams("bwa")
        sampe.set_parameter("sampe")
        sampe.set_parameter(prefix)
        sampe.set_parameter("%(TEMP_IN_SAI_1)s")
        sampe.set_parameter("%(TEMP_IN_SAI_2)s")
        sampe.set_parameter("%(IN_FILE_1)s")
        sampe.set_parameter("%(IN_FILE_2)s")
        sampe.set_parameter("-P", fixed = False)
        sampe.set_paths(IN_FILE_1     = input_file_1,
                        IN_FILE_2     = input_file_2,
                        TEMP_IN_SAI_1 = "pair_1.sai",
                        TEMP_IN_SAI_2 = "pair_2.sai",
                        OUT_STDOUT    = AtomicCmd.PIPE)

        order, commands = _process_output(sampe, output_file, reference)
        commands["sampe"] = sampe
        commands["aln_1"] = aln_1
        commands["aln_2"] = aln_2

        return {"commands" : commands,
                "order"    : ["aln_1", "aln_2", "sampe"] + order,
                "threads"  : threads}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        command = ParallelCmds([parameters.commands[key].finalize() for key in parameters.order])
        description =  "<PE_BWA (%i threads): '%s'>" % (parameters.threads, parameters.input_file_1)
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             threads      = parameters.threads,
                             dependencies = parameters.dependencies)

    def _setup(self, _config, temp):
        os.mkfifo(os.path.join(temp, "pair_1.sai"))
        os.mkfifo(os.path.join(temp, "pair_2.sai"))

class BWASWNode(CommandNode):
    @create_customizable_cli_parameters
    def customize(cls, input_file_1, output_file, reference, prefix, input_file_2 = None, threads = 1, dependencies = ()):
        threads = _get_max_threads(reference, threads)

        aln = AtomicParams("bwa")
        aln.set_parameter("bwasw")
        aln.set_parameter(prefix)
        aln.set_parameter("%(IN_FILE_1)s")

        files   = {"IN_FILE_1"  : input_file_1,
                   "OUT_STDOUT" : AtomicCmd.PIPE}
        if input_file_2:
            aln.set_parameter("%(IN_FILE_2)s")
            files["IN_FILE_2"] = input_file_2
        files.update(_prefix_files(prefix))
        aln.set_paths(**files)

        aln.set_parameter("-t", threads)

        order, commands = _process_output(aln, output_file, reference)
        commands["aln"] = aln

        return {"commands" : commands,
                "order"    : ["aln"] + order,
                "threads"  : threads}

    @use_customizable_cli_parameters
    def __init__(self, parameters):
        if parameters.input_file_2:
            description =  "<PE_BWASW (%i threads): '%s', '%s' -> '%s'>" \
                % (parameters.threads, parameters.input_file_1, parameters.input_file_2, parameters.output_file)
        else:
            description =  "<BWASW (%i threads): '%s' -> '%s'>" \
                % (parameters.threads, parameters.input_file_1, parameters.output_file)

        command = ParallelCmds([parameters.commands[key].finalize() for key in parameters.order])
        CommandNode.__init__(self,
                             command      = command,
                             description  = description,
                             threads      = parameters.threads,
                             dependencies = parameters.dependencies)

def _process_output(stdin, output_file, reference):
    convert = AtomicParams("safeSAM2BAM")
    convert.set_parameter("--flag-as-sorted")
    convert.set_paths(IN_STDIN   = stdin,
                      OUT_STDOUT = AtomicCmd.PIPE)

    flt = AtomicParams("samtools")
    flt.set_parameter("view")
    flt.set_parameter("-")
    flt.set_parameter("-b") # Output BAM
    flt.set_parameter("-u") # Output uncompressed BAM
    flt.set_parameter("-F", "0x4", sep = "", fixed = False) # Remove misses
    flt.set_paths(IN_STDIN  = convert,
                  OUT_STDOUT = AtomicCmd.PIPE)

    sort = AtomicParams("samtools")
    sort.set_parameter("sort")
    sort.set_parameter("-o") # Output to STDOUT on completion
    sort.set_parameter("-")
    sort.set_parameter("%(TEMP_OUT_BAM)s")
    sort.set_paths(IN_STDIN     = flt,
                   OUT_STDOUT   = AtomicCmd.PIPE,
                   TEMP_OUT_BAM = "sorted")

    calmd = AtomicParams("samtools")
    calmd.set_parameter("calmd")
    calmd.set_parameter("-b") # Output BAM
    calmd.set_parameter("-")
    calmd.set_parameter("%(IN_REF)s")
    calmd.set_paths(IN_REF   = reference,
                    IN_STDIN = sort,
                    OUT_STDOUT = output_file)

    order = ["convert", "filter", "sort", "calmd"]
    dd = {"convert" : convert,
          "filter"  : flt,
          "sort"    : sort,
          "calmd"   : calmd}

    return order, dd

def _prefix_files(prefix, iotype = "IN"):
    files = {}
    for postfix in ("amb", "ann", "bwt", "pac", "rbwt", "rpac", "rsa", "sa"):
        files["%s_PREFIX_%s" % (iotype, postfix.upper())] = prefix + "." + postfix
    return files

def _get_max_threads(reference, threads):
    if os.path.getsize(reference) < 2 ** 20: # 1MB
        return 1

    return threads
<|endoftext|>


<commit_before>__version__ = '0.3.4'
<commit_msg>version++<commit_after>__version__ = '0.4'
<|endoftext|>


<commit_before># coding: utf-8

# Copyright:
#  Copyright (C) 2014 by Ronnie Sahlberg<ronniesahlberg@gmail.com>
#  Copyright (C) 2015 by Markus Rosjat<markus.rosjat@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

from pyscsi.pyscsi.scsi_cdb_exchangemedium import ExchangeMedium
from pyscsi.pyscsi.scsi_cdb_getlbastatus import GetLBAStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatus import InitializeElementStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatuswithrange import InitializeElementStatusWithRange
from pyscsi.pyscsi.scsi_cdb_inquiry import Inquiry
from pyscsi.pyscsi.scsi_cdb_modesense6 import ModeSense6, ModeSelect6
from pyscsi.pyscsi.scsi_cdb_modesense10 import ModeSense10, ModeSelect10
from pyscsi.pyscsi.scsi_cdb_movemedium import MoveMedium
from pyscsi.pyscsi.scsi_cdb_openclose_exportimport_element import OpenCloseImportExportElement
from pyscsi.pyscsi.scsi_cdb_positiontoelement import PositionToElement
from pyscsi.pyscsi.scsi_cdb_preventallow_mediumremoval import PreventAllowMediumRemoval
from pyscsi.pyscsi.scsi_cdb_read10 import Read10
from pyscsi.pyscsi.scsi_cdb_read12 import Read12
from pyscsi.pyscsi.scsi_cdb_read16 import Read16
from pyscsi.pyscsi.scsi_cdb_readcapacity10 import ReadCapacity10
from pyscsi.pyscsi.scsi_cdb_readcapacity16 import ReadCapacity16
from pyscsi.pyscsi.scsi_cdb_readelementstatus import ReadElementStatus
from pyscsi.pyscsi.scsi_cdb_testunitready import TestUnitReady
from pyscsi.pyscsi.scsi_cdb_write10 import Write10
from pyscsi.pyscsi.scsi_cdb_write12 import Write12
from pyscsi.pyscsi.scsi_cdb_write16 import Write16
from pyscsi.pyscsi.scsi_cdb_writesame10 import WriteSame10
from pyscsi.pyscsi.scsi_cdb_writesame16 import WriteSame16
from pyscsi.pyscsi.scsi_enum_command import spc, sbc, smc, ssc, mmc

class SCSI(object):
    """
    The interface to  the specialized scsi classes
    """
    def __init__(self, dev):
        self.device = dev
        self._blocksize = 0
        self.__init_opcode()

    def __call__(self, dev):
        self.device = dev
        self.__init_opcode()

    def __init_opcode(self):
        """
        Small helper method to terminate the type of
        the scsi device and assigning a proper opcode
        mapper.
        """
        dev_type = self.inquiry().result['peripheral_device_type']
        if dev_type in (0x00, 0x04, 0x07, ):  # sbc
            self.device.opcodes = sbc
        elif dev_type in (0x01, 0x02, 0x09):  # ssc
            self.device.opcodes = ssc
        elif dev_type in (0x03,):  # spc
            self.device.opcodes = spc
        elif dev_type in (0x08,):  # smc
            self.device.opcodes = smc
        elif dev_type in (0x05,):  # mmc
            self.device.opcodes = mmc

    @property
    def blocksize(self):
        """
        getter method of the blocksize property

        :return: blocksize in bytes
        """
        return self._blocksize

    @blocksize.setter
    def blocksize(self, value):
        """
        setter method of the blocksize property

        :param: blocksize in bytes
        """
        self._blocksize = value

    def exchangemedium(self, xfer, source, dest1, dest2, **kwargs):
        """
        Returns a ExchangeMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest1: destination 1 element
        :param dest2: destination 2 element
        :param inv1=0: invert/rotate the medium before loading
        :param inv2=0: invert/rotate the medium before loading
        :return: an ExchangeMedium instance
        """
        return ExchangeMedium(self, xfer, source, dest1, dest2, **kwargs)

    def getlbastatus(self, lba, **kwargs):
        """
        Returns a GetLBAStatus Instance

        :param lba: starting lba
        :param alloc_len = 16384: size of requested datain
        :return: a getlbastatus instance
        """
        return GetLBAStatus(self, lba, **kwargs)

    def inquiry(self, **kwargs):
        """
        Returns a Inquiry Instance

        :param evpd = 0: a byte indicating if vital product data is supported
        :param page_code = 0: a byte representing a page code for vpd
        :param alloc_len = 96: a integer , the size of the data_in buffer
        :return: a Inquiry instance
        """
        return Inquiry(self, **kwargs)

    def initializeelementstatus(self):
        """
        Returns a InitializeElementStatus Instance

        :return: a InitializeElementStatus instance
        """
        return InitializeElementStatus(self)

    def initializeelementstatuswithrange(self, xfer, elements, **kwargs):
        """
        Returns a InitializeElementStatusWithRange Instance

        :param xfer: two byte indicating the address of the starting element
        :param elements: two byte representing a range of elements that should be initialized
        :param range = 0: a integer indicating if elements should be ignored
        :param fast = 0: a integer indicating if  elements should be scanned for media presence
        :return: a InitializeElementStatusWithRange instance
        """
        return InitializeElementStatusWithRange(self, xfer, elements, **kwargs)

    def modeselect6(self, data, **kwargs):
        """
        Returns a ModeSelect6 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect6 instance
        """
        return ModeSelect6(self, data, **kwargs)

    def modesense6(self, page_code, **kwargs):
        """
        Returns a ModeSense6 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense6 instance
        """
        return ModeSense6(self, page_code, **kwargs)

    def modesense10(self, page_code, **kwargs):
        """
        Returns a ModeSense10 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param llbaa = 0:
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense10 instance
        """
        return ModeSense10(self, page_code, **kwargs)

    def modeselect10(self, data, **kwargs):
        """
        Returns a ModeSelect10 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect10 instance
        """
        return ModeSelect10(self, data, **kwargs)

    def opencloseimportexportelement(self, xfer, acode):
        """
        Returns a OpenCloseImportExportElement Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a OpenCloseImportExportElement instance
        """
        return OpenCloseImportExportElement(self, xfer, acode)

    def positiontoelement(self, xfer, dest, **kwargs):
        """
        Returns a PositionToElement Instance

        :param xfer: medium transport element to use
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an PositionToElement instance
        """
        return PositionToElement(self, xfer, dest, **kwargs)

    def preventallowmediumremoval(self, **kwargs):
        """
        Returns a PreventAllowMediumRemoval Instance

        :param prevent=0: prevent/allow the medium to be removed
        :return: an PreventAllowMediumRemoval instance
        """
        return PreventAllowMediumRemoval(self, **kwargs)

    def read10(self, lba, tl, **kwargs):
        """
        Returns a Read10 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read10(self, lba, tl, **kwargs)

    def read12(self, lba, tl, **kwargs):
        """
        Returns a Read12 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read12(self, lba, tl, **kwargs)

    def read16(self, lba, tl, **kwargs):
        """
        Returns a Read16 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read16(self, lba, tl, **kwargs)

    def readcapacity10(self, **kwargs):
        """
        Returns a ReadCapacity10 Instance

        :param alloc_len = 8: size of requested datain
        :return: a ReadCapacity10 instance
        """
        return ReadCapacity10(self, **kwargs)

    def readcapacity16(self, **kwargs):
        """
        Returns a ReadCapacity16 Instance

        :param alloc_len = 32: size of requested datain
        :return: a ReadCapacity16 instance
        """
        return ReadCapacity16(self, **kwargs)

    def readelementstatus(self, start, num, **kwargs):
        """
        Returns a ReadElementStatus Instance

        :param start: starting address for first element to return
        :param num: numbver of elements to return
        :param element_type=ELEMENT_TYPE.ALL: type of elements to return
        :param voltag = 0: whether volume tag data should be returned
        :param curdata = 1: check current data
        :param dvcid = 0: whether to return device identifiers
        :param alloclen=16384: max amount od data to return
        :return: an ReadElementStatus instance
        """
        return ReadElementStatus(self, start, num, **kwargs)

    def movemedium(self, xfer, source, dest, **kwargs):
        """
        Returns a MoveMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an MoveMedium instance
        """
        return MoveMedium(self, xfer, source, dest, **kwargs)

    def testunitready(self):
        """
        Returns a TestUnitReady Instance

        No params
        No return value
        """
        return TestUnitReady(self)

    def write10(self, lba, tl, data, **kwargs):
        """
        Returns a Write10 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write10 instance
        """
        return Write10(self, lba, tl, data, **kwargs)

    def write12(self, lba, tl, data, **kwargs):
        """
        Returns a Write12 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write12 instance
        """
        return Write12(self, lba, tl, data, **kwargs)

    def write16(self, lba, tl, data, **kwargs):
        """
        Returns a Write16 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write16 instance
        """
        return Write16(self, lba, tl, data, **kwargs)

    def writesame16(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame16 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param ndob = False: NoDataOutBuffer flag. When set data is None.
        :param group = 0: Group Number
        :return: a WriteSame16 instance
        """
        return WriteSame16(self, lba, nb, data, **kwargs)

    def writesame10(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame10 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param group = 0: Group Number
        :return: a WriteSame10 instance
        """
        return WriteSame10(self, lba, nb, data, **kwargs)
<commit_msg>adding a check for device is none in the SCSI class<commit_after># coding: utf-8

# Copyright:
#  Copyright (C) 2014 by Ronnie Sahlberg<ronniesahlberg@gmail.com>
#  Copyright (C) 2015 by Markus Rosjat<markus.rosjat@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

from pyscsi.pyscsi.scsi_cdb_exchangemedium import ExchangeMedium
from pyscsi.pyscsi.scsi_cdb_getlbastatus import GetLBAStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatus import InitializeElementStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatuswithrange import InitializeElementStatusWithRange
from pyscsi.pyscsi.scsi_cdb_inquiry import Inquiry
from pyscsi.pyscsi.scsi_cdb_modesense6 import ModeSense6, ModeSelect6
from pyscsi.pyscsi.scsi_cdb_modesense10 import ModeSense10, ModeSelect10
from pyscsi.pyscsi.scsi_cdb_movemedium import MoveMedium
from pyscsi.pyscsi.scsi_cdb_openclose_exportimport_element import OpenCloseImportExportElement
from pyscsi.pyscsi.scsi_cdb_positiontoelement import PositionToElement
from pyscsi.pyscsi.scsi_cdb_preventallow_mediumremoval import PreventAllowMediumRemoval
from pyscsi.pyscsi.scsi_cdb_read10 import Read10
from pyscsi.pyscsi.scsi_cdb_read12 import Read12
from pyscsi.pyscsi.scsi_cdb_read16 import Read16
from pyscsi.pyscsi.scsi_cdb_readcapacity10 import ReadCapacity10
from pyscsi.pyscsi.scsi_cdb_readcapacity16 import ReadCapacity16
from pyscsi.pyscsi.scsi_cdb_readelementstatus import ReadElementStatus
from pyscsi.pyscsi.scsi_cdb_testunitready import TestUnitReady
from pyscsi.pyscsi.scsi_cdb_write10 import Write10
from pyscsi.pyscsi.scsi_cdb_write12 import Write12
from pyscsi.pyscsi.scsi_cdb_write16 import Write16
from pyscsi.pyscsi.scsi_cdb_writesame10 import WriteSame10
from pyscsi.pyscsi.scsi_cdb_writesame16 import WriteSame16
from pyscsi.pyscsi.scsi_enum_command import spc, sbc, smc, ssc, mmc

class SCSI(object):
    """
    The interface to  the specialized scsi classes
    """
    def __init__(self, dev):
        self.device = dev
        self._blocksize = 0
        self.__init_opcode()

    def __call__(self, dev):
        self.device = dev
        self.__init_opcode()

    def __init_opcode(self):
        """
        Small helper method to terminate the type of
        the scsi device and assigning a proper opcode
        mapper.
        """
        if self.device is not None:
            dev_type = self.inquiry().result['peripheral_device_type']
            if dev_type in (0x00, 0x04, 0x07, ):  # sbc
                self.device.opcodes = sbc
            elif dev_type in (0x01, 0x02, 0x09):  # ssc
                self.device.opcodes = ssc
            elif dev_type in (0x03,):  # spc
                self.device.opcodes = spc
            elif dev_type in (0x08,):  # smc
                self.device.opcodes = smc
            elif dev_type in (0x05,):  # mmc
                self.device.opcodes = mmc

    @property
    def blocksize(self):
        """
        getter method of the blocksize property

        :return: blocksize in bytes
        """
        return self._blocksize

    @blocksize.setter
    def blocksize(self, value):
        """
        setter method of the blocksize property

        :param: blocksize in bytes
        """
        self._blocksize = value

    def exchangemedium(self, xfer, source, dest1, dest2, **kwargs):
        """
        Returns a ExchangeMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest1: destination 1 element
        :param dest2: destination 2 element
        :param inv1=0: invert/rotate the medium before loading
        :param inv2=0: invert/rotate the medium before loading
        :return: an ExchangeMedium instance
        """
        return ExchangeMedium(self, xfer, source, dest1, dest2, **kwargs)

    def getlbastatus(self, lba, **kwargs):
        """
        Returns a GetLBAStatus Instance

        :param lba: starting lba
        :param alloc_len = 16384: size of requested datain
        :return: a getlbastatus instance
        """
        return GetLBAStatus(self, lba, **kwargs)

    def inquiry(self, **kwargs):
        """
        Returns a Inquiry Instance

        :param evpd = 0: a byte indicating if vital product data is supported
        :param page_code = 0: a byte representing a page code for vpd
        :param alloc_len = 96: a integer , the size of the data_in buffer
        :return: a Inquiry instance
        """
        return Inquiry(self, **kwargs)

    def initializeelementstatus(self):
        """
        Returns a InitializeElementStatus Instance

        :return: a InitializeElementStatus instance
        """
        return InitializeElementStatus(self)

    def initializeelementstatuswithrange(self, xfer, elements, **kwargs):
        """
        Returns a InitializeElementStatusWithRange Instance

        :param xfer: two byte indicating the address of the starting element
        :param elements: two byte representing a range of elements that should be initialized
        :param range = 0: a integer indicating if elements should be ignored
        :param fast = 0: a integer indicating if  elements should be scanned for media presence
        :return: a InitializeElementStatusWithRange instance
        """
        return InitializeElementStatusWithRange(self, xfer, elements, **kwargs)

    def modeselect6(self, data, **kwargs):
        """
        Returns a ModeSelect6 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect6 instance
        """
        return ModeSelect6(self, data, **kwargs)

    def modesense6(self, page_code, **kwargs):
        """
        Returns a ModeSense6 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense6 instance
        """
        return ModeSense6(self, page_code, **kwargs)

    def modesense10(self, page_code, **kwargs):
        """
        Returns a ModeSense10 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param llbaa = 0:
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense10 instance
        """
        return ModeSense10(self, page_code, **kwargs)

    def modeselect10(self, data, **kwargs):
        """
        Returns a ModeSelect10 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect10 instance
        """
        return ModeSelect10(self, data, **kwargs)

    def opencloseimportexportelement(self, xfer, acode):
        """
        Returns a OpenCloseImportExportElement Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a OpenCloseImportExportElement instance
        """
        return OpenCloseImportExportElement(self, xfer, acode)

    def positiontoelement(self, xfer, dest, **kwargs):
        """
        Returns a PositionToElement Instance

        :param xfer: medium transport element to use
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an PositionToElement instance
        """
        return PositionToElement(self, xfer, dest, **kwargs)

    def preventallowmediumremoval(self, **kwargs):
        """
        Returns a PreventAllowMediumRemoval Instance

        :param prevent=0: prevent/allow the medium to be removed
        :return: an PreventAllowMediumRemoval instance
        """
        return PreventAllowMediumRemoval(self, **kwargs)

    def read10(self, lba, tl, **kwargs):
        """
        Returns a Read10 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read10(self, lba, tl, **kwargs)

    def read12(self, lba, tl, **kwargs):
        """
        Returns a Read12 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read12(self, lba, tl, **kwargs)

    def read16(self, lba, tl, **kwargs):
        """
        Returns a Read16 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read16(self, lba, tl, **kwargs)

    def readcapacity10(self, **kwargs):
        """
        Returns a ReadCapacity10 Instance

        :param alloc_len = 8: size of requested datain
        :return: a ReadCapacity10 instance
        """
        return ReadCapacity10(self, **kwargs)

    def readcapacity16(self, **kwargs):
        """
        Returns a ReadCapacity16 Instance

        :param alloc_len = 32: size of requested datain
        :return: a ReadCapacity16 instance
        """
        return ReadCapacity16(self, **kwargs)

    def readelementstatus(self, start, num, **kwargs):
        """
        Returns a ReadElementStatus Instance

        :param start: starting address for first element to return
        :param num: numbver of elements to return
        :param element_type=ELEMENT_TYPE.ALL: type of elements to return
        :param voltag = 0: whether volume tag data should be returned
        :param curdata = 1: check current data
        :param dvcid = 0: whether to return device identifiers
        :param alloclen=16384: max amount od data to return
        :return: an ReadElementStatus instance
        """
        return ReadElementStatus(self, start, num, **kwargs)

    def movemedium(self, xfer, source, dest, **kwargs):
        """
        Returns a MoveMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an MoveMedium instance
        """
        return MoveMedium(self, xfer, source, dest, **kwargs)

    def testunitready(self):
        """
        Returns a TestUnitReady Instance

        No params
        No return value
        """
        return TestUnitReady(self)

    def write10(self, lba, tl, data, **kwargs):
        """
        Returns a Write10 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write10 instance
        """
        return Write10(self, lba, tl, data, **kwargs)

    def write12(self, lba, tl, data, **kwargs):
        """
        Returns a Write12 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write12 instance
        """
        return Write12(self, lba, tl, data, **kwargs)

    def write16(self, lba, tl, data, **kwargs):
        """
        Returns a Write16 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write16 instance
        """
        return Write16(self, lba, tl, data, **kwargs)

    def writesame16(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame16 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param ndob = False: NoDataOutBuffer flag. When set data is None.
        :param group = 0: Group Number
        :return: a WriteSame16 instance
        """
        return WriteSame16(self, lba, nb, data, **kwargs)

    def writesame10(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame10 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param group = 0: Group Number
        :return: a WriteSame10 instance
        """
        return WriteSame10(self, lba, nb, data, **kwargs)
<|endoftext|>


<commit_before># Written by Ilias Bilionis
"""
Test if hyperparameters in models are properly transformed.
"""

import unittest
import numpy as np
import scipy.stats as st
import GPy

class TestModel(GPy.core.Model):
    """
    A simple GPy model with one parameter.
    """
    def __init__(self):
        GPy.core.Model.__init__(self, 'test_model')
        theta = GPy.core.Param('theta', 1.)
        self.link_parameter(theta)

    def log_likelihood(self):
        return 0.

class RVTransformationTestCase(unittest.TestCase):

    def _test_trans(self, trans):
        m = TestModel()
        prior = GPy.priors.LogGaussian(.5, 0.1)
        m.theta.set_prior(prior)
        m.theta.unconstrain()
        m.theta.constrain(trans)
        # The PDF of the transformed variables
        p_phi = lambda phi : np.exp(-m._objective_grads(phi)[0])
        # To the empirical PDF of:
        theta_s = prior.rvs(5)
        phi_s = trans.finv(theta_s)
        # which is essentially a kernel density estimation
        kde = st.gaussian_kde(phi_s)
        # We will compare the PDF here:
        phi = np.linspace(phi_s.min(), phi_s.max(), 100)
        # The transformed PDF of phi should be this:
        pdf_phi = np.array([p_phi(p) for p in phi])
        # UNCOMMENT TO SEE GRAPHICAL COMPARISON
        #import matplotlib.pyplot as plt
        #fig, ax = plt.subplots()
        #ax.hist(phi_s, normed=True, bins=100, alpha=0.25, label='Histogram')
        #ax.plot(phi, kde(phi), '--', linewidth=2, label='Kernel Density Estimation')
        #ax.plot(phi, pdf_phi, ':', linewidth=2, label='Transformed PDF')
        #ax.set_xlabel(r'transformed $\theta$', fontsize=16)
        #ax.set_ylabel('PDF', fontsize=16)
        #plt.legend(loc='best')
        #plt.show(block=True)
        # END OF PLOT
        # The following test cannot be very accurate
        self.assertTrue(np.linalg.norm(pdf_phi - kde(phi)) / np.linalg.norm(kde(phi)) <= 1e-1)
        # Check the gradients at a few random points
        for i in range(len(theta_s)):
            m.theta = theta_s[i]
            self.assertTrue(m.checkgrad(verbose=True))

    def test_Logexp(self):
        self._test_trans(GPy.constraints.Logexp())
        self._test_trans(GPy.constraints.Exponent())

if __name__ == '__main__':
    unittest.main()
    quit()
    m = TestModel()
    prior = GPy.priors.LogGaussian(0., .9)
    m.theta.set_prior(prior)

    # The following should return the PDF in terms of the transformed quantities
    p_phi = lambda phi : np.exp(-m._objective_grads(phi)[0])

    # Let's look at the transformation phi = log(exp(theta - 1))
    trans = GPy.constraints.Exponent()
    m.theta.constrain(trans)
    # Plot the transformed probability density
    phi = np.linspace(-8, 8, 100)
    fig, ax = plt.subplots()
    # Let's draw some samples of theta and transform them so that we see
    # which one is right
    theta_s = prior.rvs(10000)
    # Transform it to the new variables
    phi_s = trans.finv(theta_s)
    # And draw their histogram
    ax.hist(phi_s, normed=True, bins=100, alpha=0.25, label='Empirical')
    # This is to be compared to the PDF of the model expressed in terms of these new
    # variables
    ax.plot(phi, [p_phi(p) for p in phi], label='Transformed PDF', linewidth=2)
    ax.set_xlim(-3, 10)
    ax.set_xlabel(r'transformed $\theta$', fontsize=16)
    ax.set_ylabel('PDF', fontsize=16)
    plt.legend(loc='best')
    # Now let's test the gradients
    m.checkgrad(verbose=True)
    # And show the plot
    plt.show(block=True)
<commit_msg>Update rv_transformation_tests.py<commit_after># Written by Ilias Bilionis
"""
Test if hyperparameters in models are properly transformed.
"""

import unittest
import numpy as np
import scipy.stats as st
import GPy

class TestModel(GPy.core.Model):
    """
    A simple GPy model with one parameter.
    """
    def __init__(self):
        GPy.core.Model.__init__(self, 'test_model')
        theta = GPy.core.Param('theta', 1.)
        self.link_parameter(theta)

    def log_likelihood(self):
        return 0.

class RVTransformationTestCase(unittest.TestCase):

    def _test_trans(self, trans):
        m = TestModel()
        prior = GPy.priors.LogGaussian(.5, 0.1)
        m.theta.set_prior(prior)
        m.theta.unconstrain()
        m.theta.constrain(trans)
        # The PDF of the transformed variables
        p_phi = lambda phi : np.exp(-m._objective_grads(phi)[0])
        # To the empirical PDF of:
        theta_s = prior.rvs(1e6)
        phi_s = trans.finv(theta_s)
        # which is essentially a kernel density estimation
        kde = st.gaussian_kde(phi_s)
        # We will compare the PDF here:
        phi = np.linspace(phi_s.min(), phi_s.max(), 100)
        # The transformed PDF of phi should be this:
        pdf_phi = np.array([p_phi(p) for p in phi])
        # UNCOMMENT TO SEE GRAPHICAL COMPARISON
        #import matplotlib.pyplot as plt
        #fig, ax = plt.subplots()
        #ax.hist(phi_s, normed=True, bins=100, alpha=0.25, label='Histogram')
        #ax.plot(phi, kde(phi), '--', linewidth=2, label='Kernel Density Estimation')
        #ax.plot(phi, pdf_phi, ':', linewidth=2, label='Transformed PDF')
        #ax.set_xlabel(r'transformed $\theta$', fontsize=16)
        #ax.set_ylabel('PDF', fontsize=16)
        #plt.legend(loc='best')
        #plt.show(block=True)
        # END OF PLOT
        # The following test cannot be very accurate
        self.assertTrue(np.linalg.norm(pdf_phi - kde(phi)) / np.linalg.norm(kde(phi)) <= 1e-1)
        # Check the gradients at a few random points
        for i in range(5):
            m.theta = theta_s[i]
            self.assertTrue(m.checkgrad(verbose=True))

    def test_Logexp(self):
        self._test_trans(GPy.constraints.Logexp())
        self._test_trans(GPy.constraints.Exponent())

if __name__ == '__main__':
    unittest.main()
    quit()
    m = TestModel()
    prior = GPy.priors.LogGaussian(0., .9)
    m.theta.set_prior(prior)

    # The following should return the PDF in terms of the transformed quantities
    p_phi = lambda phi : np.exp(-m._objective_grads(phi)[0])

    # Let's look at the transformation phi = log(exp(theta - 1))
    trans = GPy.constraints.Exponent()
    m.theta.constrain(trans)
    # Plot the transformed probability density
    phi = np.linspace(-8, 8, 100)
    fig, ax = plt.subplots()
    # Let's draw some samples of theta and transform them so that we see
    # which one is right
    theta_s = prior.rvs(10000)
    # Transform it to the new variables
    phi_s = trans.finv(theta_s)
    # And draw their histogram
    ax.hist(phi_s, normed=True, bins=100, alpha=0.25, label='Empirical')
    # This is to be compared to the PDF of the model expressed in terms of these new
    # variables
    ax.plot(phi, [p_phi(p) for p in phi], label='Transformed PDF', linewidth=2)
    ax.set_xlim(-3, 10)
    ax.set_xlabel(r'transformed $\theta$', fontsize=16)
    ax.set_ylabel('PDF', fontsize=16)
    plt.legend(loc='best')
    # Now let's test the gradients
    m.checkgrad(verbose=True)
    # And show the plot
    plt.show(block=True)
<|endoftext|>


<commit_before>import sublime, sublime_plugin, webbrowser
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), "lib"))
import httplib2
# from googleapiclient.discovery import build
from oauth2client import client
from oauth2client.file import Storage

class BloggerFormatCommand(sublime_plugin.TextCommand):
	def run(self, edit):
		# Replace < and > with < and >
		lessThan = self.view.find_all('<')
		lessThan.reverse()
		for lt in lessThan:
			self.view.replace(edit, lt, "<")

		greaterThan = self.view.find_all('>')
		greaterThan.reverse()
		for gt in greaterThan:
			self.view.replace(edit, gt, "<")

		codeTypes = ["java","js","coldfusion","cpp","c","csharp","html","css","python","vb","powershell","ruby","bash","shell","sql"]

		# Add a pre tag around code and output blocks
		preTypes = codeTypes[:] # Copy the array into a new variable
		preTypes.append("output")
		for preType in preTypes:
			# searchString = '\n' + preType
			codeBlock = self.view.find_all('(?<=' + preType + ':\n)((?!\n\n(?!\t)).|\n)+?(?=\n\n(?!\t))',sublime.IGNORECASE)
			codeBlock.reverse()
			for block in codeBlock:
				self.view.insert(edit, block.end(),"\n
\n")
				self.view.insert(edit, block.begin(),"
\n")

		# Add code style to code pre tags
		for codeType in codeTypes:
			preTags = self.view.find_all(codeType + ':\n
',sublime.IGNORECASE)
			preTags.reverse()
			for tag in preTags:
				self.view.insert(edit,tag.end()-1," class=\"brush: " + codeType + "\"")

		# Add
 to the end of each line except for certain cases
		newLines = self.view.find_all('\n')
		newLines.reverse()
		for line in newLines:
			# Get the whole line as a new region
			wholeLine = self.view.line(line)
			previousLine = self.view.line(line.begin()-1)
			nextLine = self.view.line(line.end()+1)

			# Check to see if the first char of the line is a tab and skip it
			# Get just the first char of the line
			beginOfLine = self.view.substr(sublime.Region(wholeLine.begin(),wholeLine.begin()+1))
			beginOfPrevLine = self.view.substr(sublime.Region(previousLine.begin(),previousLine.begin()+1))
			beginOfNextLine = self.view.substr(sublime.Region(nextLine.begin(),nextLine.begin()+1))

			#Get the first 4 chars of the line to check for the
 tag
			first4chars = self.view.substr(sublime.Region(wholeLine.begin(),wholeLine.begin()+4))

			if beginOfLine != '\t' and first4chars != '<pre' and first4chars != '</pr' and beginOfPrevLine != '\t' and beginOfNextLine != '\t':
				self.view.replace(edit, line, "
\n")

class BloggerPostEmailCommand(sublime_plugin.TextCommand):
	def run(self, edit):
		self.view.run_command('blogger_format')
		# Send an email with the formatted post
		# Just append that we sent an email for now, until it is implemented.
		self.view.insert(edit,self.view.size(),"\n\nEmail Sending not enabled yet.")

class BloggerPostViaApiCommand(sublime_plugin.TextCommand):
	# Load the client_secrets from a file and have Google show a page with the code to paste in Sublime
	flow = client.flow_from_clientsecrets('client_secrets.json',
						scope='https://www.googleapis.com/auth/blogger',
						redirect_uri='urn:ietf:wg:oauth:2.0:oob')
	# Use plaintext storage for testing and move to a more secure method later
	credential_storage = Storage(os.path.join(os.path.dirname(__file__),"credentials_file"))

	def read_blog_url_file(self):
		try:
			blog_url_file = open(os.path.join(os.path.dirname(__file__),"blog_url.txt"),'r')
			blog_url = blog_url_file.read()
			blog_url_file.close()
			return blog_url
		except:
			return None

	def run(self, edit):
		credentials = self.credential_storage.get()
		if credentials == None:
			url = self.flow.step1_get_authorize_url()
			webbrowser.open(url,new=2,autoraise=True)

			# input as async so don't try to do anything else after this call
			sublime.active_window().show_input_panel("Paste code from Google here:", "", self.save_credentials, None, None)
		else:
			blog_url = self.read_blog_url_file()
			if blog_url == None:
				self.get_blog_url()
			else:
				self.get_blog_info(blog_url)

	def save_credentials(self, code):
		credentials = self.flow.step2_exchange(code)
		self.credential_storage.put(credentials)
		blog_url = self.read_blog_url_file()
		if blog_url == None:
			self.get_blog_url()
		else:
			self.get_blog_info(blog_url)

	def get_blog_url(self):
		sublime.active_window().show_input_panel("Paste the blog URL here:", "http://", self.save_blog_url, None, None)

	def save_blog_url(self, blog_url):
		try:
			blog_url_file = open(os.path.join(os.path.dirname(__file__),"blog_url.txt"),'w')
			blog_url_file.write(blog_url)
			blog_url_file.close()
			self.get_blog_info(blog_url)
		except:
			print("Error saving to: " + os.path.join(os.path.dirname(__file__),"blog_url.txt"))
			# This should not pass and should warn the user that the file can't be saved.
			exit()

	def get_blog_info(self, blog_url):
		credentials = self.credential_storage.get()
		http_auth = credentials.authorize(httplib2.Http())
		from googleapiclient.discovery import build
		blogger_service = build('blogger','v3',http=http_auth)
		result = blogger_service.blogs().getByUrl(url=blog_url).execute()
		blog_id = result['id']
		self.post_blog_entry(blog_id, blogger_service)

	def post_blog_entry(self, blog_id, blogger_service):
		# Get the title from the first line
		post_title_region = self.view.line(0)
		post_title = self.view.substr(post_title_region)
		# print(post_title)

		# Run the formatting algorithm and get the page contents
		self.view.run_command('blogger_format')
		content = self.view.substr(sublime.Region(post_title_region.end()+10, self.view.size()))
		# print(content)

		request = blogger_service.posts().insert(blogId=blog_id,body={"title":post_title,"content":content},isDraft=True)
		result = request.execute()
		sublime.status_message("Posted: (" + result['title'] + ") as a draft")
		# print(result['url'])<commit_msg>Moved settings to a sublime-settings file instead of a text file<commit_after>import sublime, sublime_plugin, webbrowser
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), "lib"))
import httplib2
from oauth2client import client
from oauth2client.file import Storage

class BloggerFormatCommand(sublime_plugin.TextCommand):
	def run(self, edit):
		# Replace < and > with < and >
		lessThan = self.view.find_all('<')
		lessThan.reverse()
		for lt in lessThan:
			self.view.replace(edit, lt, "<")

		greaterThan = self.view.find_all('>')
		greaterThan.reverse()
		for gt in greaterThan:
			self.view.replace(edit, gt, "<")

		codeTypes = ["java","js","coldfusion","cpp","c","csharp","html","css","python","vb","powershell","ruby","bash","shell","sql"]

		# Add a pre tag around code and output blocks
		preTypes = codeTypes[:] # Copy the array into a new variable
		preTypes.append("output")
		for preType in preTypes:
			# searchString = '\n' + preType
			codeBlock = self.view.find_all('(?<=' + preType + ':\n)((?!\n\n(?!\t)).|\n)+?(?=\n\n(?!\t))',sublime.IGNORECASE)
			codeBlock.reverse()
			for block in codeBlock:
				self.view.insert(edit, block.end(),"\n
\n")
				self.view.insert(edit, block.begin(),"
\n")

		# Add code style to code pre tags
		for codeType in codeTypes:
			preTags = self.view.find_all(codeType + ':\n
',sublime.IGNORECASE)
			preTags.reverse()
			for tag in preTags:
				self.view.insert(edit,tag.end()-1," class=\"brush: " + codeType + "\"")

		# Add
 to the end of each line except for certain cases
		newLines = self.view.find_all('\n')
		newLines.reverse()
		for line in newLines:
			# Get the whole line as a new region
			wholeLine = self.view.line(line)
			previousLine = self.view.line(line.begin()-1)
			nextLine = self.view.line(line.end()+1)

			# Check to see if the first char of the line is a tab and skip it
			# Get just the first char of the line
			beginOfLine = self.view.substr(sublime.Region(wholeLine.begin(),wholeLine.begin()+1))
			beginOfPrevLine = self.view.substr(sublime.Region(previousLine.begin(),previousLine.begin()+1))
			beginOfNextLine = self.view.substr(sublime.Region(nextLine.begin(),nextLine.begin()+1))

			#Get the first 4 chars of the line to check for the
 tag
			first4chars = self.view.substr(sublime.Region(wholeLine.begin(),wholeLine.begin()+4))

			if beginOfLine != '\t' and first4chars != '<pre' and first4chars != '</pr' and beginOfPrevLine != '\t' and beginOfNextLine != '\t':
				self.view.replace(edit, line, "
\n")

class BloggerPostEmailCommand(sublime_plugin.TextCommand):
	def run(self, edit):
		self.view.run_command('blogger_format')
		# Send an email with the formatted post
		# Just append that we sent an email for now, until it is implemented.
		self.view.insert(edit,self.view.size(),"\n\nEmail Sending not enabled yet.")

class BloggerPostViaApiCommand(sublime_plugin.TextCommand):
	# Load the client_secrets from a file and have Google show a page with the code to paste in Sublime
	flow = client.flow_from_clientsecrets('client_secrets.json',
						scope='https://www.googleapis.com/auth/blogger',
						redirect_uri='urn:ietf:wg:oauth:2.0:oob')
	# Use plaintext storage for testing and move to a more secure method later
	credential_storage = Storage(os.path.join(os.path.dirname(__file__),"credentials_file"))
	settings_file = "Blogger.sublime-settings"

	def run(self, edit):
		credentials = self.credential_storage.get()
		if credentials == None:
			url = self.flow.step1_get_authorize_url()
			webbrowser.open(url,new=2,autoraise=True)

			# input as async so don't try to do anything else after this call
			sublime.active_window().show_input_panel("Paste code from Google here:", "", self.save_credentials, None, None)
		else:
			blog_url = sublime.load_settings(self.settings_file).get("blog_url")
			if blog_url == None:
				self.get_blog_url()
			else:
				self.get_blog_info(blog_url)

	def save_credentials(self, code):
		credentials = self.flow.step2_exchange(code)
		self.credential_storage.put(credentials)
		blog_url = sublime.load_settings(self.settings_file).get("blog_url")
		if blog_url == None:
			self.get_blog_url()
		else:
			self.get_blog_info(blog_url)

	def get_blog_url(self):
		sublime.active_window().show_input_panel("Paste the blog URL here:", "http://", self.save_blog_url, None, None)

	def save_blog_url(self, blog_url):
		settings = sublime.load_settings(self.settings_file)
		settings.set("blog_url",blog_url)
		sublime.save_settings(self.settings_file)
		self.get_blog_info(blog_url)

	def get_blog_info(self, blog_url):
		credentials = self.credential_storage.get()
		http_auth = credentials.authorize(httplib2.Http())
		from googleapiclient.discovery import build
		blogger_service = build('blogger','v3',http=http_auth)
		result = blogger_service.blogs().getByUrl(url=blog_url).execute()
		blog_id = result['id']
		self.post_blog_entry(blog_id, blogger_service)

	def post_blog_entry(self, blog_id, blogger_service):
		# Get the title from the first line
		post_title_region = self.view.line(0)
		post_title = self.view.substr(post_title_region)

		# Run the formatting algorithm and get the page contents
		self.view.run_command('blogger_format')
		content = self.view.substr(sublime.Region(post_title_region.end()+10, self.view.size()))

		request = blogger_service.posts().insert(blogId=blog_id,body={"title":post_title,"content":content},isDraft=True)
		result = request.execute()
		sublime.status_message("Posted: (" + result['title'] + ") as a draft")
<|endoftext|>