<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
import subprocess
import os
import os.path
from urllib import unquote

from module.utils import html_unescape
from module.plugins.Hoster import Hoster

def which(program):
    """Works exactly like the unix command which

    Courtesy of http://stackoverflow.com/a/377028/675646"""

    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None

class YoutubeCom(Hoster):
    __name__ = "YoutubeCom"
    __type__ = "hoster"
    __pattern__ = r"https?://(?:[^/]*?)youtube\.com/watch.*?[?&]v=.*"
    __version__ = "0.34"
    __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting", "hd"),
                  ("fmt", "int", "FMT/ITAG Number (5-102, 0 for auto)", 0),
                  (".mp4", "bool", "Allow .mp4", True),
                  (".flv", "bool", "Allow .flv", True),
                  (".webm", "bool", "Allow .webm", False),
                  (".3gp", "bool", "Allow .3gp", False),
                  ("3d", "bool", "Prefer 3D", False)]
    __description__ = """Youtube.com Video Download Hoster"""
    __author_name__ = ("spoob", "zoidberg")
    __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz")

    # name, width, height, quality ranking, 3D
    formats = {5: (".flv", 400, 240, 1, False),
               6: (".flv", 640, 400, 4, False),
               17: (".3gp", 176, 144, 0, False),
               18: (".mp4", 480, 360, 2, False),
               22: (".mp4", 1280, 720, 8, False),
               43: (".webm", 640, 360, 3, False),
               34: (".flv", 640, 360, 4, False),
               35: (".flv", 854, 480, 6, False),
               36: (".3gp", 400, 240, 1, False),
               37: (".mp4", 1920, 1080, 9, False),
               38: (".mp4", 4096, 3072, 10, False),
               44: (".webm", 854, 480, 5, False),
               45: (".webm", 1280, 720, 7, False),
               46: (".webm", 1920, 1080, 9, False),
               82: (".mp4", 640, 360, 3, True),
               83: (".mp4", 400, 240, 1, True),
               84: (".mp4", 1280, 720, 8, True),
               85: (".mp4", 1920, 1080, 9, True),
               100: (".webm", 640, 360, 3, True),
               101: (".webm", 640, 360, 4, True),
               102: (".webm", 1280, 720, 8, True)}

    def setup(self):
        self.resumeDownload = self.multiDL = True

    def process(self, pyfile):
        html = self.load(pyfile.url, decode=True)

        if re.search(r'
', html):
            self.offline()

        if "We have been receiving a large volume of requests from your network." in html:
            self.tempOffline()

        #get config
        use3d = self.getConfig("3d")
        if use3d:
            quality = {"sd": 82, "hd": 84, "fullhd": 85, "240p": 83, "360p": 82,
                       "480p": 82, "720p": 84, "1080p": 85, "3072p": 85}
        else:
            quality = {"sd": 18, "hd": 22, "fullhd": 37, "240p": 5, "360p": 18,
                       "480p": 35, "720p": 22, "1080p": 37, "3072p": 38}
        desired_fmt = self.getConfig("fmt")
        if desired_fmt and desired_fmt not in self.formats:
            self.logWarning("FMT %d unknown - using default." % desired_fmt)
            desired_fmt = 0
        if not desired_fmt:
            desired_fmt = quality.get(self.getConfig("quality"), 18)

        #parse available streams
        streams = re.search(r'"url_encoded_fmt_stream_map": "(.*?)",', html).group(1)
        streams = [x.split('\u0026') for x in streams.split(',')]
        streams = [dict((y.split('=', 1)) for y in x) for x in streams]
        streams = [(int(x['itag']), "%s&signature=%s" % (unquote(x['url']), x['sig'])) for x in streams]
        #self.logDebug("Found links: %s" % streams)
        self.logDebug("AVAILABLE STREAMS: %s" % [x[0] for x in streams])

        #build dictionary of supported itags (3D/2D)
        allowed = lambda x: self.getConfig(self.formats[x][0])
        streams = [x for x in streams if x[0] in self.formats and allowed(x[0])]
        if not streams:
            self.fail("No available stream meets your preferences")
        fmt_dict = dict([x for x in streams if self.formats[x[0]][4] == use3d] or streams)

        self.logDebug("DESIRED STREAM: ITAG:%d (%s) %sfound, %sallowed" %
                      (desired_fmt, "%s %dx%d Q:%d 3D:%s" % self.formats[desired_fmt],
                       "" if desired_fmt in fmt_dict else "NOT ", "" if allowed(desired_fmt) else "NOT "))

        #return fmt nearest to quality index
        if desired_fmt in fmt_dict and allowed(desired_fmt):
            fmt = desired_fmt
        else:
            sel = lambda x: self.formats[x][3]  # select quality index
            comp = lambda x, y: abs(sel(x) - sel(y))

            self.logDebug("Choosing nearest fmt: %s" % [(x, allowed(x), comp(x, desired_fmt)) for x in fmt_dict.keys()])
            fmt = reduce(lambda x, y: x if comp(x, desired_fmt) <= comp(y, desired_fmt) and
                         sel(x) > sel(y) else y, fmt_dict.keys())

        self.logDebug("Chosen fmt: %s" % fmt)
        url = fmt_dict[fmt]
        self.logDebug("URL: %s" % url)

        #set file name
        file_suffix = self.formats[fmt][0] if fmt in self.formats else ".flv"
        file_name_pattern = '<meta name="title" content="(.+?)">'
        name = re.search(file_name_pattern, html).group(1).replace("/", "")
        pyfile.name = html_unescape(name)

        time = re.search(r"t=((\d+)m)?(\d+)s", pyfile.url)
        ffmpeg = which("ffmpeg")
        if ffmpeg and time:
            m, s = time.groups()[1:]
            if not m:
                m = "0"

            pyfile.name += " (starting at %s:%s)" % (m, s)
        pyfile.name += file_suffix

        filename = self.download(url)

        if ffmpeg and time:
            inputfile = filename + "_"
            os.rename(filename, inputfile)

            subprocess.call([
                ffmpeg,
                "-ss", "00:%s:%s" % (m, s),
                "-i", inputfile,
                "-vcodec", "copy",
                "-acodec", "copy",
                filename])
            os.remove(inputfile)
<commit_msg>YouTube: fixed #287<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
import subprocess
import os
from urllib import unquote

from module.utils import html_unescape
from module.plugins.Hoster import Hoster

def which(program):
    """Works exactly like the unix command which

    Courtesy of http://stackoverflow.com/a/377028/675646"""

    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None

class YoutubeCom(Hoster):
    __name__ = "YoutubeCom"
    __type__ = "hoster"
    __pattern__ = r"https?://(?:[^/]*?)youtube\.com/watch.*?[?&]v=.*"
    __version__ = "0.35"
    __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting", "hd"),
                  ("fmt", "int", "FMT/ITAG Number (5-102, 0 for auto)", 0),
                  (".mp4", "bool", "Allow .mp4", True),
                  (".flv", "bool", "Allow .flv", True),
                  (".webm", "bool", "Allow .webm", False),
                  (".3gp", "bool", "Allow .3gp", False),
                  ("3d", "bool", "Prefer 3D", False)]
    __description__ = """Youtube.com Video Download Hoster"""
    __author_name__ = ("spoob", "zoidberg")
    __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz")

    # Invalid characters that must be removed from the file name
    invalidChars = ':?><"|\\'

    # name, width, height, quality ranking, 3D
    formats = {5: (".flv", 400, 240, 1, False),
               6: (".flv", 640, 400, 4, False),
               17: (".3gp", 176, 144, 0, False),
               18: (".mp4", 480, 360, 2, False),
               22: (".mp4", 1280, 720, 8, False),
               43: (".webm", 640, 360, 3, False),
               34: (".flv", 640, 360, 4, False),
               35: (".flv", 854, 480, 6, False),
               36: (".3gp", 400, 240, 1, False),
               37: (".mp4", 1920, 1080, 9, False),
               38: (".mp4", 4096, 3072, 10, False),
               44: (".webm", 854, 480, 5, False),
               45: (".webm", 1280, 720, 7, False),
               46: (".webm", 1920, 1080, 9, False),
               82: (".mp4", 640, 360, 3, True),
               83: (".mp4", 400, 240, 1, True),
               84: (".mp4", 1280, 720, 8, True),
               85: (".mp4", 1920, 1080, 9, True),
               100: (".webm", 640, 360, 3, True),
               101: (".webm", 640, 360, 4, True),
               102: (".webm", 1280, 720, 8, True)}

    def setup(self):
        self.resumeDownload = self.multiDL = True

    def process(self, pyfile):
        html = self.load(pyfile.url, decode=True)

        if re.search(r'
', html):
            self.offline()

        if "We have been receiving a large volume of requests from your network." in html:
            self.tempOffline()

        #get config
        use3d = self.getConfig("3d")
        if use3d:
            quality = {"sd": 82, "hd": 84, "fullhd": 85, "240p": 83, "360p": 82,
                       "480p": 82, "720p": 84, "1080p": 85, "3072p": 85}
        else:
            quality = {"sd": 18, "hd": 22, "fullhd": 37, "240p": 5, "360p": 18,
                       "480p": 35, "720p": 22, "1080p": 37, "3072p": 38}
        desired_fmt = self.getConfig("fmt")
        if desired_fmt and desired_fmt not in self.formats:
            self.logWarning("FMT %d unknown - using default." % desired_fmt)
            desired_fmt = 0
        if not desired_fmt:
            desired_fmt = quality.get(self.getConfig("quality"), 18)

        #parse available streams
        streams = re.search(r'"url_encoded_fmt_stream_map": "(.*?)",', html).group(1)
        streams = [x.split('\u0026') for x in streams.split(',')]
        streams = [dict((y.split('=', 1)) for y in x) for x in streams]
        streams = [(int(x['itag']), "%s&signature=%s" % (unquote(x['url']), x['sig'])) for x in streams]
        #self.logDebug("Found links: %s" % streams)
        self.logDebug("AVAILABLE STREAMS: %s" % [x[0] for x in streams])

        #build dictionary of supported itags (3D/2D)
        allowed = lambda x: self.getConfig(self.formats[x][0])
        streams = [x for x in streams if x[0] in self.formats and allowed(x[0])]
        if not streams:
            self.fail("No available stream meets your preferences")
        fmt_dict = dict([x for x in streams if self.formats[x[0]][4] == use3d] or streams)

        self.logDebug("DESIRED STREAM: ITAG:%d (%s) %sfound, %sallowed" %
                      (desired_fmt, "%s %dx%d Q:%d 3D:%s" % self.formats[desired_fmt],
                       "" if desired_fmt in fmt_dict else "NOT ", "" if allowed(desired_fmt) else "NOT "))

        #return fmt nearest to quality index
        if desired_fmt in fmt_dict and allowed(desired_fmt):
            fmt = desired_fmt
        else:
            sel = lambda x: self.formats[x][3]  # select quality index
            comp = lambda x, y: abs(sel(x) - sel(y))

            self.logDebug("Choosing nearest fmt: %s" % [(x, allowed(x), comp(x, desired_fmt)) for x in fmt_dict.keys()])
            fmt = reduce(lambda x, y: x if comp(x, desired_fmt) <= comp(y, desired_fmt) and
                         sel(x) > sel(y) else y, fmt_dict.keys())

        self.logDebug("Chosen fmt: %s" % fmt)
        url = fmt_dict[fmt]
        self.logDebug("URL: %s" % url)

        #set file name
        file_suffix = self.formats[fmt][0] if fmt in self.formats else ".flv"
        file_name_pattern = '<meta name="title" content="(.+?)">'
        name = re.search(file_name_pattern, html).group(1).replace("/", "")

        # Cleaning invalid characters from the file name
        for c in self.invalidChars:
            name = name.replace(c, '_')

        pyfile.name = html_unescape(name)

        time = re.search(r"t=((\d+)m)?(\d+)s", pyfile.url)
        ffmpeg = which("ffmpeg")
        if ffmpeg and time:
            m, s = time.groups()[1:]
            if not m:
                m = "0"

            pyfile.name += " (starting at %s:%s)" % (m, s)
        pyfile.name += file_suffix

        filename = self.download(url)

        if ffmpeg and time:
            inputfile = filename + "_"
            os.rename(filename, inputfile)

            subprocess.call([
                ffmpeg,
                "-ss", "00:%s:%s" % (m, s),
                "-i", inputfile,
                "-vcodec", "copy",
                "-acodec", "copy",
                filename])
            os.remove(inputfile)
<|endoftext|>


<commit_before>VERSION = '3.20.1'
<commit_msg>start working on 3.20.2.dev<commit_after>VERSION = '3.20.2.dev'
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division

import six
import warnings
import tempfile
from astropy.io import fits
from astropy.wcs import WCS
from astropy import units as u
import numpy as np
from radio_beam import Beam, Beams

from .. import SpectralCube, StokesSpectralCube, BooleanArrayMask, LazyMask, VaryingResolutionSpectralCube
from .. import cube_utils
from .. utils import BeamWarning

# Read and write from a CASA image. This has a few
# complications. First, by default CASA does not return the
# "python order" and so we either have to transpose the cube on
# read or have dueling conventions. Second, CASA often has
# degenerate stokes axes present in unpredictable places (3rd or
# 4th without a clear expectation). We need to replicate these
# when writing but don't want them in memory. By default, try to
# yield the same array in memory that we would get from astropy.

def is_casa_image(input, **kwargs):
    if isinstance(input, six.string_types):
        if input.endswith('.image'):
            return True
    return False

def wcs_casa2astropy(ia, coordsys):
    """
    Convert a casac.coordsys object into an astropy.wcs.WCS object
    """

    # Rather than try and parse the CASA coordsys ourselves, we delegate
    # to CASA by getting it to write out a FITS file and reading back in
    # using WCS

    from casatasks import exportfits

    tmpimagefile = tempfile.mktemp() + '.image'
    tmpfitsfile = tempfile.mktemp() + '.fits'
    ia.newimagefromarray(outfile=tmpimagefile,
                         pixels=np.ones([1] * coordsys.naxes()),
                         csys=coordsys.torecord(), log=False)
    exportfits(tmpimagefile, tmpfitsfile)

    return WCS(tmpfitsfile)

def load_casa_image(filename, skipdata=False,
                    skipvalid=False, skipcs=False, **kwargs):
    """
    Load a cube (into memory?) from a CASA image. By default it will transpose
    the cube into a 'python' order and drop degenerate axes. These options can
    be suppressed. The object holds the coordsys object from the image in
    memory.
    """

    try:
        import casatools
        ia = casatools.image()
    except ImportError:
        try:
            from taskinit import iatool
            ia = iatool()
        except ImportError:
            raise ImportError("Could not import CASA (casac) and therefore cannot read CASA .image files")

    # use the ia tool to get the file contents
    try:
        ia.open(filename)
    except AssertionError as ex:
        if 'must be of cReqPath type' in str(ex):
            raise IOError("File {0} not found.  Error was: {1}"
                          .format(filename, str(ex)))
        else:
            raise ex

    # read in the data
    if not skipdata:
        # CASA data are apparently transposed.
        data = ia.getchunk().transpose()

    # CASA stores validity of data as a mask
    if not skipvalid:
        valid = ia.getchunk(getmask=True).transpose()

    # transpose is dealt with within the cube object

    # read in coordinate system object
    casa_cs = ia.coordsys()

    wcs = wcs_casa2astropy(ia, casa_cs)

    unit = ia.brightnessunit()

    beam_ = ia.restoringbeam()
    if 'major' in beam_:
        beam = Beam(major=u.Quantity(beam_['major']['value'], unit=beam_['major']['unit']),
                    minor=u.Quantity(beam_['minor']['value'], unit=beam_['minor']['unit']),
                    pa=u.Quantity(beam_['positionangle']['value'], unit=beam_['positionangle']['unit']),
                   )
    elif 'beams' in beam_:
        bdict = beam_['beams']
        if beam_['nStokes'] > 1:
            raise NotImplementedError()
        nbeams = len(bdict)
        assert nbeams == beam_['nChannels']
        stokesidx = '*0'

        majors = [u.Quantity(bdict['*{0}'.format(ii)][stokesidx]['major']['value'],
                             bdict['*{0}'.format(ii)][stokesidx]['major']['unit']) for ii in range(nbeams)]
        minors = [u.Quantity(bdict['*{0}'.format(ii)][stokesidx]['minor']['value'],
                             bdict['*{0}'.format(ii)][stokesidx]['minor']['unit']) for ii in range(nbeams)]
        pas = [u.Quantity(bdict['*{0}'.format(ii)][stokesidx]['positionangle']['value'],
                          bdict['*{0}'.format(ii)][stokesidx]['positionangle']['unit']) for ii in range(nbeams)]

        beams = Beams(major=u.Quantity(majors),
                      minor=u.Quantity(minors),
                      pa=u.Quantity(pas))
    else:
        warnings.warn("No beam information found in CASA image.",
                      BeamWarning)

    # don't need this yet
    # stokes = get_casa_axis(temp_cs, wanttype="Stokes", skipdeg=False,)

    #    if stokes == None:
    #        order = np.arange(self.data.ndim)
    #    else:
    #        order = []
    #        for ax in np.arange(self.data.ndim+1):
    #            if ax == stokes:
    #                continue
    #            order.append(ax)

    #    self.casa_cs = ia.coordsys(order)

        # This should work, but coordsys.reorder() has a bug
        # on the error checking. JIRA filed. Until then the
        # axes will be reversed from the original.

        # if transpose == True:
        #    new_order = np.arange(self.data.ndim)
        #    new_order = new_order[-1*np.arange(self.data.ndim)-1]
        #    print new_order
        #    self.casa_cs.reorder(new_order)

    # close the ia tool
    ia.close()

    meta = {'filename': filename,
            'BUNIT': unit}

    if wcs.naxis == 3:
        mask = BooleanArrayMask(np.logical_not(valid), wcs)
        if 'beam' in locals():
            cube = SpectralCube(data, wcs, mask, meta=meta, beam=beam)
        elif 'beams' in locals():
            cube = VaryingResolutionSpectralCube(data, wcs, mask, meta=meta, beams=beams)
        else:
            cube = SpectralCube(data, wcs, mask, meta=meta)
        # we've already loaded the cube into memory because of CASA
        # limitations, so there's no reason to disallow operations
        cube.allow_huge_operations = True

    elif wcs.naxis == 4:
        data, wcs = cube_utils._split_stokes(data, wcs)
        mask = {}
        for component in data:
            data_, wcs_slice = cube_utils._orient(data[component], wcs)
            mask[component] = LazyMask(np.isfinite, data=data[component],
                                       wcs=wcs_slice)

            if 'beam' in locals():
                data[component] = SpectralCube(data_, wcs_slice, mask[component],
                                               meta=meta, beam=beam)
            elif 'beams' in locals():
                data[component] = VaryingResolutionSpectralCube(data_,
                                                                wcs_slice,
                                                                mask[component],
                                                                meta=meta,
                                                                beams=beams)
            else:
                data[component] = SpectralCube(data_, wcs_slice, mask[component],
                                               meta=meta)

            data[component].allow_huge_operations = True

        cube = StokesSpectralCube(stokes_data=data)

    return cube
<commit_msg>Don't put Stokes axis last<commit_after>from __future__ import print_function, absolute_import, division

import six
import warnings
import tempfile
from astropy.io import fits
from astropy.wcs import WCS
from astropy import units as u
import numpy as np
from radio_beam import Beam, Beams

from .. import SpectralCube, StokesSpectralCube, BooleanArrayMask, LazyMask, VaryingResolutionSpectralCube
from .. import cube_utils
from .. utils import BeamWarning

# Read and write from a CASA image. This has a few
# complications. First, by default CASA does not return the
# "python order" and so we either have to transpose the cube on
# read or have dueling conventions. Second, CASA often has
# degenerate stokes axes present in unpredictable places (3rd or
# 4th without a clear expectation). We need to replicate these
# when writing but don't want them in memory. By default, try to
# yield the same array in memory that we would get from astropy.

def is_casa_image(input, **kwargs):
    if isinstance(input, six.string_types):
        if input.endswith('.image'):
            return True
    return False

def wcs_casa2astropy(ia, coordsys):
    """
    Convert a casac.coordsys object into an astropy.wcs.WCS object
    """

    # Rather than try and parse the CASA coordsys ourselves, we delegate
    # to CASA by getting it to write out a FITS file and reading back in
    # using WCS

    from casatasks import exportfits

    tmpimagefile = tempfile.mktemp() + '.image'
    tmpfitsfile = tempfile.mktemp() + '.fits'
    ia.newimagefromarray(outfile=tmpimagefile,
                         pixels=np.ones([1] * coordsys.naxes()),
                         csys=coordsys.torecord(), log=False)
    exportfits(tmpimagefile, tmpfitsfile, stokeslast=False)

    return WCS(tmpfitsfile)

def load_casa_image(filename, skipdata=False,
                    skipvalid=False, skipcs=False, **kwargs):
    """
    Load a cube (into memory?) from a CASA image. By default it will transpose
    the cube into a 'python' order and drop degenerate axes. These options can
    be suppressed. The object holds the coordsys object from the image in
    memory.
    """

    try:
        import casatools
        ia = casatools.image()
    except ImportError:
        try:
            from taskinit import iatool
            ia = iatool()
        except ImportError:
            raise ImportError("Could not import CASA (casac) and therefore cannot read CASA .image files")

    # use the ia tool to get the file contents
    try:
        ia.open(filename)
    except AssertionError as ex:
        if 'must be of cReqPath type' in str(ex):
            raise IOError("File {0} not found.  Error was: {1}"
                          .format(filename, str(ex)))
        else:
            raise ex

    # read in the data
    if not skipdata:
        # CASA data are apparently transposed.
        data = ia.getchunk().transpose()

    # CASA stores validity of data as a mask
    if not skipvalid:
        valid = ia.getchunk(getmask=True).transpose()

    # transpose is dealt with within the cube object

    # read in coordinate system object
    casa_cs = ia.coordsys()

    wcs = wcs_casa2astropy(ia, casa_cs)

    unit = ia.brightnessunit()

    beam_ = ia.restoringbeam()
    if 'major' in beam_:
        beam = Beam(major=u.Quantity(beam_['major']['value'], unit=beam_['major']['unit']),
                    minor=u.Quantity(beam_['minor']['value'], unit=beam_['minor']['unit']),
                    pa=u.Quantity(beam_['positionangle']['value'], unit=beam_['positionangle']['unit']),
                   )
    elif 'beams' in beam_:
        bdict = beam_['beams']
        if beam_['nStokes'] > 1:
            raise NotImplementedError()
        nbeams = len(bdict)
        assert nbeams == beam_['nChannels']
        stokesidx = '*0'

        majors = [u.Quantity(bdict['*{0}'.format(ii)][stokesidx]['major']['value'],
                             bdict['*{0}'.format(ii)][stokesidx]['major']['unit']) for ii in range(nbeams)]
        minors = [u.Quantity(bdict['*{0}'.format(ii)][stokesidx]['minor']['value'],
                             bdict['*{0}'.format(ii)][stokesidx]['minor']['unit']) for ii in range(nbeams)]
        pas = [u.Quantity(bdict['*{0}'.format(ii)][stokesidx]['positionangle']['value'],
                          bdict['*{0}'.format(ii)][stokesidx]['positionangle']['unit']) for ii in range(nbeams)]

        beams = Beams(major=u.Quantity(majors),
                      minor=u.Quantity(minors),
                      pa=u.Quantity(pas))
    else:
        warnings.warn("No beam information found in CASA image.",
                      BeamWarning)

    # don't need this yet
    # stokes = get_casa_axis(temp_cs, wanttype="Stokes", skipdeg=False,)

    #    if stokes == None:
    #        order = np.arange(self.data.ndim)
    #    else:
    #        order = []
    #        for ax in np.arange(self.data.ndim+1):
    #            if ax == stokes:
    #                continue
    #            order.append(ax)

    #    self.casa_cs = ia.coordsys(order)

        # This should work, but coordsys.reorder() has a bug
        # on the error checking. JIRA filed. Until then the
        # axes will be reversed from the original.

        # if transpose == True:
        #    new_order = np.arange(self.data.ndim)
        #    new_order = new_order[-1*np.arange(self.data.ndim)-1]
        #    print new_order
        #    self.casa_cs.reorder(new_order)

    # close the ia tool
    ia.close()

    meta = {'filename': filename,
            'BUNIT': unit}

    if wcs.naxis == 3:
        mask = BooleanArrayMask(np.logical_not(valid), wcs)
        if 'beam' in locals():
            cube = SpectralCube(data, wcs, mask, meta=meta, beam=beam)
        elif 'beams' in locals():
            cube = VaryingResolutionSpectralCube(data, wcs, mask, meta=meta, beams=beams)
        else:
            cube = SpectralCube(data, wcs, mask, meta=meta)
        # we've already loaded the cube into memory because of CASA
        # limitations, so there's no reason to disallow operations
        cube.allow_huge_operations = True

    elif wcs.naxis == 4:
        data, wcs = cube_utils._split_stokes(data, wcs)
        mask = {}
        for component in data:
            data_, wcs_slice = cube_utils._orient(data[component], wcs)
            mask[component] = LazyMask(np.isfinite, data=data[component],
                                       wcs=wcs_slice)

            if 'beam' in locals():
                data[component] = SpectralCube(data_, wcs_slice, mask[component],
                                               meta=meta, beam=beam)
            elif 'beams' in locals():
                data[component] = VaryingResolutionSpectralCube(data_,
                                                                wcs_slice,
                                                                mask[component],
                                                                meta=meta,
                                                                beams=beams)
            else:
                data[component] = SpectralCube(data_, wcs_slice, mask[component],
                                               meta=meta)

            data[component].allow_huge_operations = True

        cube = StokesSpectralCube(stokes_data=data)

    return cube
<|endoftext|>


<commit_before>"""The ok.py script runs tests, checks for updates, and saves your work.

Common uses:
  python3 ok.py          Run unlocked tests (and save your work).
  python3 ok.py -u       Unlock new tests interactively.
  python3 ok.py -h       Display full help documentation.

This script will search the current directory for test files. Make sure that
ok.py appears in the same directory as the assignment you wish to test.
Otherwise, use -t to specify a test file manually.
"""

# TODO(denero) Add mechanism for removing DEVELOPER INSTRUCTIONS.
DEVELOPER_INSTRUCTIONS = """

This multi-line string contains instructions for developers. It is removed
when the client is distributed to students.

This file is responsible for coordinating all communication with the ok-server.
Students should never need to modify this file.

Local and remote interactions are encapsulated as protocols.
Contributors should do the following to add a protocol to this autograder:

    1- Extend the Protocol class and implement on_start and on_interact.
    2- Add the classname of your protocol to the "protocols" list.
    3- If your protocol needs command line arguments, change parse_input.

A standard protocol lifecycle has only one round-trip communication with the
server, processed by the on_start method. If other interactions are required
outside of this lifecycle, the send_to_server function can be used to send and
receive information from the server outside of the default times. Such
communications should be limited to the body of an on_interact method.
"""
BACKUP_FILE = ".ok_messages"

from client import config
from client.models import *
from client.protocols import *
from client.utils import auth
from client.utils import loading
from client.utils import output
from datetime import datetime
from urllib import request, error
import client
import argparse
import base64
import json
import multiprocessing
import pickle
import sys
import time

def send_to_server(access_token, messages, name, server,
                   insecure=False):
    """Send messages to server, along with user authentication."""
    data = {
        'assignment': name,
        'messages': messages,
    }
    try:
        prefix = "http" if insecure else "https"
        address = prefix + '://' + server + '/api/v1/submission'
        serialized = json.dumps(data).encode(encoding='utf-8')
        # TODO(denero) Wrap in timeout (maybe use PR #51 timed execution).
        # TODO(denero) Send access token with the request
        address += "?access_token={0}&client_version={1}".format(
            access_token, client.__version__)
        req = request.Request(address)
        req.add_header("Content-Type", "application/json")
        response = request.urlopen(req, serialized)
        return json.loads(response.read().decode('utf-8'))
    except error.HTTPError as ex:
        # print("Error while sending to server: {}".format(ex))
        try:
            if ex.code == 403:
                response = ex.read().decode('utf-8')
                response_json = json.loads(response)
                get_latest_version(response_json['data']['download_link'])
            #message = response_json['message']
            #indented = '\n'.join('\t' + line for line in message.split('\n'))
            #print(indented)
            return {}
        except Exception as e:
            # print(e)
            # print("Couldn't connect to server")
            pass

def dump_to_server(access_token, msg_queue, name, server, insecure, staging_queue):
    while not msg_queue.empty():
        message = msg_queue.get()
        staging_queue.put(message)
        try:
            if send_to_server(access_token, message, name, server, insecure) == None:
                staging_queue.get() #throw away successful message
        except error.URLError as ex:
            pass
    return

#####################
# Software Updating #
#####################

def get_latest_version(download_link):
    """Check for the latest version of ok and update this file accordingly.
    """
    #print("We detected that you are running an old version of ok.py: {0}".format(VERSION))

    # Get server version

    try:
        req = request.Request(download_link)
        response = request.urlopen(req)

        zip_binary = response.read()
        with open('ok', 'wb') as f:
            f.write(zip_binary)
        #print("Done updating!")
    except error.HTTPError:
        # print("Error when downloading new version")
        pass

##########################
# Command-line Interface #
##########################

def parse_input():
    """Parses command line input."""
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-q', '--question', type=str,
                        help="focus on a specific question")
    parser.add_argument('-s', '--server', type=str,
                        default='ok-server.appspot.com',
                        help="server address")
    parser.add_argument('-t', '--tests', metavar='A', default='tests', type=str,
                        help="partial name or path to test file or directory")
    parser.add_argument('-u', '--unlock', action='store_true',
                        help="unlock tests interactively")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="print more output")
    parser.add_argument('--insecure', action='store_true',
                        help="uses http instead of https")
    parser.add_argument('-i', '--interactive', action='store_true',
                        help="toggle interactive mode")
    parser.add_argument('-l', '--lock', type=str,
                        help="partial path to directory to lock")
    parser.add_argument('-f', '--force', action='store_true',
                        help="wait for server response without timeout")
    parser.add_argument('-a', '--authenticate', action='store_true',
                        help="authenticate, ignoring previous authentication")
    parser.add_argument('--local', action='store_true',
                        help="disable any network activity")
    parser.add_argument('--timeout', type=int, default=10,
                        help="set the timeout duration for running tests")
    parser.add_argument('--version', action='store_true',
                        help="Prints the version number and quits")
    parser.add_argument('--score', action='store_true',
                        help="Scores the assignment")
    return parser.parse_args()

def server_timer():
    """Timeout for the server."""
    time.sleep(0.8)

def main():
    """Run all relevant aspects of ok.py."""
    args = parse_input()

    if args.version:
        print("okpy=={}".format(client.__version__))
        exit(0)

    if not args.local and not args.insecure:
        try:
            import ssl
        except:
            sys.exit("SSL Bindings are not installed. You can install python3 SSL bindings or \nrun ok locally with python3 ok --local")

    server_thread, timer_thread = None, None
    try:
        print("You are running version {0} of ok.py".format(client.__version__))
        if not args.local:
            timer_thread = multiprocessing.Process(target=server_timer, args=())
            timer_thread.start()
        cases = {case.type: case for case in core.get_testcases(config.cases)}
        assignment = loading.load_tests(args.tests, cases)

        logger = sys.stdout = output.OutputLogger()

        protocols = [p(args, assignment, logger)
                     for p in protocol.get_protocols(config.protocols)]

        messages = dict()
        msg_queue = multiprocessing.Queue()
        file_contents = []

        try:
            with open(BACKUP_FILE, 'rb') as fp:
                file_contents = pickle.load(fp)
        except IOError as e:
            # File doesn't exist, so file_contents should be empty
            pass
        except EOFError as e:
            # File is empty, so no messages are inside
            pass

        for message in file_contents:
            msg_queue.put(message)

        for proto in protocols:
            messages[proto.name] = proto.on_start()
        messages['timestamp'] = str(datetime.now())

        if not args.local:
            try:
                access_token = auth.authenticate(args.authenticate)
                msg_queue.put(messages)
                staging_queue = multiprocessing.Queue()
                server_thread = multiprocessing.Process(
                    target=dump_to_server,
                    args=(access_token, msg_queue, assignment['name'],
                          args.server, args.insecure, staging_queue))
                server_thread.start()
            except error.URLError as ex:
                # TODO(soumya) Make a better error message
                # print("Nothing was sent to the server!")
                pass

        interact_msg = {}

        for proto in protocols:
            interact_msg[proto.name] = proto.on_interact()

        interact_msg['timestamp'] = str(datetime.now())

        # TODO(denero) Print server responses.

        # TODO(albert): a premature error might prevent tests from being
        # dumped. Perhaps add this in a "finally" clause.
        loading.dump_tests(args.tests, assignment)

        if not args.local:
            msg_queue.put(interact_msg)

            while timer_thread.is_alive():
                pass

            if not args.force:
                server_thread.terminate()
            else:
                server_thread.join()

            dump_list = []
            while not msg_queue.empty():
                dump_list.append(msg_queue.get_nowait())
            while not staging_queue.empty():
                dump_list.append(staging_queue.get_nowait())
            with open(BACKUP_FILE, 'wb') as fp:
                pickle.dump(dump_list, fp)

    except KeyboardInterrupt:
        if timer_thread:
            timer_thread.terminate()
        if server_thread:
            server_thread.terminate()

if __name__ == '__main__':
    main()
<commit_msg>Change force to submit<commit_after>"""The ok.py script runs tests, checks for updates, and saves your work.

Common uses:
  python3 ok.py          Run unlocked tests (and save your work).
  python3 ok.py -u       Unlock new tests interactively.
  python3 ok.py -h       Display full help documentation.

This script will search the current directory for test files. Make sure that
ok.py appears in the same directory as the assignment you wish to test.
Otherwise, use -t to specify a test file manually.
"""

# TODO(denero) Add mechanism for removing DEVELOPER INSTRUCTIONS.
DEVELOPER_INSTRUCTIONS = """

This multi-line string contains instructions for developers. It is removed
when the client is distributed to students.

This file is responsible for coordinating all communication with the ok-server.
Students should never need to modify this file.

Local and remote interactions are encapsulated as protocols.
Contributors should do the following to add a protocol to this autograder:

    1- Extend the Protocol class and implement on_start and on_interact.
    2- Add the classname of your protocol to the "protocols" list.
    3- If your protocol needs command line arguments, change parse_input.

A standard protocol lifecycle has only one round-trip communication with the
server, processed by the on_start method. If other interactions are required
outside of this lifecycle, the send_to_server function can be used to send and
receive information from the server outside of the default times. Such
communications should be limited to the body of an on_interact method.
"""
BACKUP_FILE = ".ok_messages"

from client import config
from client.models import *
from client.protocols import *
from client.utils import auth
from client.utils import loading
from client.utils import output
from datetime import datetime
from urllib import request, error
import client
import argparse
import base64
import json
import multiprocessing
import pickle
import sys
import time

def send_to_server(access_token, messages, name, server,
                   insecure=False):
    """Send messages to server, along with user authentication."""
    data = {
        'assignment': name,
        'messages': messages,
    }
    try:
        prefix = "http" if insecure else "https"
        address = prefix + '://' + server + '/api/v1/submission'
        serialized = json.dumps(data).encode(encoding='utf-8')
        # TODO(denero) Wrap in timeout (maybe use PR #51 timed execution).
        # TODO(denero) Send access token with the request
        address += "?access_token={0}&client_version={1}".format(
            access_token, client.__version__)
        req = request.Request(address)
        req.add_header("Content-Type", "application/json")
        response = request.urlopen(req, serialized)
        return json.loads(response.read().decode('utf-8'))
    except error.HTTPError as ex:
        # print("Error while sending to server: {}".format(ex))
        try:
            if ex.code == 403:
                response = ex.read().decode('utf-8')
                response_json = json.loads(response)
                get_latest_version(response_json['data']['download_link'])
            #message = response_json['message']
            #indented = '\n'.join('\t' + line for line in message.split('\n'))
            #print(indented)
            return {}
        except Exception as e:
            # print(e)
            # print("Couldn't connect to server")
            pass

def dump_to_server(access_token, msg_queue, name, server, insecure, staging_queue):
    while not msg_queue.empty():
        message = msg_queue.get()
        staging_queue.put(message)
        try:
            if send_to_server(access_token, message, name, server, insecure) == None:
                staging_queue.get() #throw away successful message
        except error.URLError as ex:
            pass
    return

#####################
# Software Updating #
#####################

def get_latest_version(download_link):
    """Check for the latest version of ok and update this file accordingly.
    """
    #print("We detected that you are running an old version of ok.py: {0}".format(VERSION))

    # Get server version

    try:
        req = request.Request(download_link)
        response = request.urlopen(req)

        zip_binary = response.read()
        with open('ok', 'wb') as f:
            f.write(zip_binary)
        #print("Done updating!")
    except error.HTTPError:
        # print("Error when downloading new version")
        pass

##########################
# Command-line Interface #
##########################

def parse_input():
    """Parses command line input."""
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-q', '--question', type=str,
                        help="focus on a specific question")
    parser.add_argument('-s', '--server', type=str,
                        default='ok-server.appspot.com',
                        help="server address")
    parser.add_argument('-t', '--tests', metavar='A', default='tests', type=str,
                        help="partial name or path to test file or directory")
    parser.add_argument('-u', '--unlock', action='store_true',
                        help="unlock tests interactively")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="print more output")
    parser.add_argument('--insecure', action='store_true',
                        help="uses http instead of https")
    parser.add_argument('-i', '--interactive', action='store_true',
                        help="toggle interactive mode")
    parser.add_argument('-l', '--lock', type=str,
                        help="partial path to directory to lock")
    parser.add_argument('--submit', action='store_true',
                        help="wait for server response without timeout")
    parser.add_argument('-a', '--authenticate', action='store_true',
                        help="authenticate, ignoring previous authentication")
    parser.add_argument('--local', action='store_true',
                        help="disable any network activity")
    parser.add_argument('--timeout', type=int, default=10,
                        help="set the timeout duration for running tests")
    parser.add_argument('--version', action='store_true',
                        help="Prints the version number and quits")
    parser.add_argument('--score', action='store_true',
                        help="Scores the assignment")
    return parser.parse_args()

def server_timer():
    """Timeout for the server."""
    time.sleep(0.8)

def main():
    """Run all relevant aspects of ok.py."""
    args = parse_input()

    if args.version:
        print("okpy=={}".format(client.__version__))
        exit(0)

    if not args.local and not args.insecure:
        try:
            import ssl
        except:
            sys.exit("SSL Bindings are not installed. You can install python3 SSL bindings or \nrun ok locally with python3 ok --local")

    server_thread, timer_thread = None, None
    try:
        print("You are running version {0} of ok.py".format(client.__version__))
        if not args.local:
            timer_thread = multiprocessing.Process(target=server_timer, args=())
            timer_thread.start()
        cases = {case.type: case for case in core.get_testcases(config.cases)}
        assignment = loading.load_tests(args.tests, cases)

        logger = sys.stdout = output.OutputLogger()

        protocols = [p(args, assignment, logger)
                     for p in protocol.get_protocols(config.protocols)]

        messages = dict()
        msg_queue = multiprocessing.Queue()
        file_contents = []

        try:
            with open(BACKUP_FILE, 'rb') as fp:
                file_contents = pickle.load(fp)
        except IOError as e:
            # File doesn't exist, so file_contents should be empty
            pass
        except EOFError as e:
            # File is empty, so no messages are inside
            pass

        for message in file_contents:
            msg_queue.put(message)

        for proto in protocols:
            messages[proto.name] = proto.on_start()
        messages['timestamp'] = str(datetime.now())

        if not args.local:
            try:
                access_token = auth.authenticate(args.authenticate)
                msg_queue.put(messages)
                staging_queue = multiprocessing.Queue()
                server_thread = multiprocessing.Process(
                    target=dump_to_server,
                    args=(access_token, msg_queue, assignment['name'],
                          args.server, args.insecure, staging_queue))
                server_thread.start()
            except error.URLError as ex:
                # TODO(soumya) Make a better error message
                # print("Nothing was sent to the server!")
                pass

        interact_msg = {}

        for proto in protocols:
            interact_msg[proto.name] = proto.on_interact()

        interact_msg['timestamp'] = str(datetime.now())

        # TODO(denero) Print server responses.

        # TODO(albert): a premature error might prevent tests from being
        # dumped. Perhaps add this in a "finally" clause.
        loading.dump_tests(args.tests, assignment)

        if not args.local:
            msg_queue.put(interact_msg)

            while timer_thread.is_alive():
                pass

            if not args.submit:
                server_thread.terminate()
            else:
                server_thread.join()

            dump_list = []
            while not msg_queue.empty():
                dump_list.append(msg_queue.get_nowait())
            while not staging_queue.empty():
                dump_list.append(staging_queue.get_nowait())
            with open(BACKUP_FILE, 'wb') as fp:
                pickle.dump(dump_list, fp)

    except KeyboardInterrupt:
        if timer_thread:
            timer_thread.terminate()
        if server_thread:
            server_thread.terminate()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2014:
#    Gabes Jean, naparuba@gmail.com
#    Gerhard Lausser, Gerhard.Lausser@consol.de
#    Gregory Starck, g.starck@gmail.com
#    Hartmut Goebel, h.goebel@goebel-consult.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

from shinken.misc.filter  import only_related_to
from shinken.misc.sorter import hst_srv_sort
from shinken.objects.service import Service
from shinken.log import logger

### Will be populated by the UI with it's own value
app = None

import time
import re
import json

# Our page
def get_page():
    return get_view('problems')

# Our page
def get_all():
    return get_view('all')

# Our page
def get_v1():
    return get_view('problems')

# Our View code. We will get different data from all and /problems
# but it's mainly filtering changes
def get_view(page):

    user = app.get_user_auth()
    if not user:
        app.bottle.redirect("/user/login")

    print 'DUMP COMMON GET', app.request.GET.__dict__

    # Look for the toolbar pref
    toolbar_pref = app.get_user_preference(user, 'toolbar')
    # If void, create an empty one
    if not toolbar_pref:
        app.set_user_preference(user, 'toolbar', 'show')
        toolbar_pref = 'show'
    toolbar = app.request.GET.get('toolbar', '')
    print "Toolbar", toolbar_pref, toolbar
    if toolbar != toolbar_pref and len(toolbar) > 0:
        print "Need to change user prefs for Toolbar",
        app.set_user_preference(user, 'toolbar', toolbar)
    toolbar_pref = app.get_user_preference(user, 'toolbar')

    # We want to limit the number of elements
    start = int(app.request.GET.get('start', '0'))
    end = int(app.request.GET.get('end', '30'))

    # We will keep a trace of our filters
    filters = {}
    ts = ['hst_srv', 'hg', 'realm', 'htag', 'stag', 'ack', 'downtime', 'crit']
    for t in ts:
        filters[t] = []

    search = app.request.GET.getall('search')
    if search == []:
        search = app.request.GET.get('global_search', '')

    # Most of the case, search will be a simple string, if so
    # make it a list of this string
    if isinstance(search, basestring):
        search = [search]

    search_str = '&'.join(search)
    print 'Search str=', search_str
    print 'And search', search

    # Load the bookmarks
    bookmarks_r = app.get_user_preference(user, 'bookmarks')
    if not bookmarks_r:
        app.set_user_preference(user, 'bookmarks', '[]')
        bookmarks_r = '[]'
    bookmarks = json.loads(bookmarks_r)
    bookmarks_ro = app.get_common_preference('bookmarks')
    if not bookmarks_ro:
        bookmarks_ro = '[]'

    bookmarksro = json.loads(bookmarks_ro)
    bookmarks = json.loads(bookmarks_r)

    items = []
    if page == 'problems':
        items = app.get_all_problems(user, to_sort=True, get_acknowledged=False)
    elif page == 'all':
        items = app.get_all_hosts_and_services(user)
    else:
        app.bottle.redirect("/problems")

    logger.info("[%s] problems", app.name)
    for i in items:
        logger.info("[%s] problems, item: %s", app.name, i.get_full_name())

    # Filter with the user interests
    # my_items = only_related_to(items, user)
    my_items = items

    # Check for related host contacts
    if not user.is_admin:
      for i in items:
        if isinstance(i,Service):
          if user in i.host.contacts:
            my_items.append(i)
            continue

    items = my_items
    logger.info("[%s] problems after user filtering", app.name)
    for i in items:
        logger.info("[%s] problems, item: %s", app.name, i.get_full_name())

    # Ok, if needed, apply the search filter
    for s in search:
        s = s.strip()
        if not s:
            continue

        print "SEARCHING FOR", s
        print "Before filtering", len(items)

        elts = s.split(':', 1)
        t = 'hst_srv'
        if len(elts) > 1:
            t = elts[0]
            s = elts[1]

        print 'Search for type %s and pattern %s' % (t, s)
        if not t in filters:
            filters[t] = []
        filters[t].append(s)

        if t == 'hst_srv':
            # We compile the pattern
            pat = re.compile(s, re.IGNORECASE)
            new_items = []
            for i in items:
                if pat.search(i.get_full_name()):
                    new_items.append(i)
                    continue
                to_add = False
                for imp in i.impacts:
                    if pat.search(imp.get_full_name()):
                        to_add = True
                for src in i.source_problems:
                    if pat.search(src.get_full_name()):
                        to_add = True
                if to_add:
                    new_items.append(i)

            items = new_items

        if t == 'hg':
            hg = app.datamgr.get_hostgroup(s)
            print 'And a valid hg filtering for', s
            items = [i for i in items if hg in i.get_hostgroups()]

        if t == 'realm':
            r = app.datamgr.get_realm(s)
            print 'Add a realm filter', r
            items = [i for i in items if i.get_realm() == r]

        if t == 'htag':
            print 'Add a htag filter', s
            items = [i for i in items if s in i.get_host_tags()]

        if t == 'stag':
            print 'Add a stag filter', s
            items = [i for i in items if i.__class__.my_type == 'service' and s in i.get_service_tags()]

        if t == 'ack':
            print "Got an ack filter", s
            if s == 'false':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or not i.problem_has_been_acknowledged]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (not i.problem_has_been_acknowledged and not i.host.problem_has_been_acknowledged)]
            if s == 'true':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or i.problem_has_been_acknowledged]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (i.problem_has_been_acknowledged or i.host.problem_has_been_acknowledged)]

        if t == 'downtime':
            print "Got an downtime filter", s
            if s == 'false':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or not i.in_scheduled_downtime]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (not i.in_scheduled_downtime and not i.host.in_scheduled_downtime)]
            if s == 'true':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or i.in_scheduled_downtime]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (i.in_scheduled_downtime or i.host.in_scheduled_downtime)]

        if t == 'crit':
            print "Add a criticity filter", s
            items = [i for i in items if (i.__class__.my_type == 'service' and i.state_id == 2) or (i.__class__.my_type == 'host' and i.state_id == 1)]

        print "After filtering for", t, s, 'we got', len(items)

    # If we are in the /problems and we do not have an ack filter
    # we apply by default the ack:false one
    print "Late problem filtering?", page == 'problems', len(filters['ack']) == 0
    if page == 'problems' and len(filters['ack']) == 0:
        # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
        items = [i for i in items if i.__class__.my_type == 'service' or not i.problem_has_been_acknowledged]
        # Now ok for hosts, but look for services, and service hosts
        items = [i for i in items if i.__class__.my_type == 'host' or (not i.problem_has_been_acknowledged and not i.host.problem_has_been_acknowledged)]

    # If we are in the /problems and we do not have an downtime filter
    # we apply by default the downtime:false one
    print "Late problem filtering?", page == 'problems', len(filters['downtime']) == 0
    if page == 'problems' and len(filters['downtime']) == 0:
        # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
        items = [i for i in items if i.__class__.my_type == 'service' or not i.in_scheduled_downtime]
        # Now ok for hosts, but look for services, and service hosts
        items = [i for i in items if i.__class__.my_type == 'host' or (not i.in_scheduled_downtime and not i.host.in_scheduled_downtime)]

    logger.info("[%s] problems after search filtering", app.name)
    for i in items:
        logger.info("[%s] problems, item: %s, %d, %d", app.name, i.get_full_name(), i.business_impact, i.state_id)

    # Now sort it!
    items.sort(hst_srv_sort)

    logger.info("[%s] problems after host/service sort", app.name)
    for i in items:
        logger.info("[%s] problems, item: %s, %d, %d", app.name, i.get_full_name(), i.business_impact, i.state_id)

    total = len(items)
    # If we overflow, came back as normal
    if start > total:
        start = 0
        end = 30

    navi = app.helper.get_navi(total, start, step=30)
    items = items[start:end]

    return {'app': app, 'pbs': items, 'user': user, 'navi': navi, 'search': search_str, 'page': page, 'filters': filters, 'bookmarks': bookmarks, 'bookmarksro': bookmarksro, 'toolbar': toolbar_pref }

# Our page
def get_pbs_widget():

    user = app.get_user_auth()
    if not user:
        app.bottle.redirect("/user/login")

    # We want to limit the number of elements, The user will be able to increase it
    nb_elements = max(0, int(app.request.GET.get('nb_elements', '10')))
    search = app.request.GET.get('search', '')

    pbs = app.datamgr.get_all_problems(to_sort=False)

    # Filter with the user interests
    pbs = only_related_to(pbs, user)

    # Sort it now
    pbs.sort(hst_srv_sort)

    # Ok, if need, appli the search filter
    if search:
        print "SEARCHING FOR", search
        print "Before filtering", len(pbs)
        # We compile the pattern
        pat = re.compile(search, re.IGNORECASE)
        new_pbs = []
        for p in pbs:
            if pat.search(p.get_full_name()):
                new_pbs.append(p)
                continue
            to_add = False
            for imp in p.impacts:
                if pat.search(imp.get_full_name()):
                    to_add = True
            for src in p.source_problems:
                if pat.search(src.get_full_name()):
                    to_add = True
            if to_add:
                new_pbs.append(p)

        pbs = new_pbs[:nb_elements]
        print "After filtering", len(pbs)

    pbs = pbs[:nb_elements]

    wid = app.request.GET.get('wid', 'widget_problems_' + str(int(time.time())))
    collapsed = (app.request.GET.get('collapsed', 'False') == 'True')

    options = {'search': {'value': search, 'type': 'text', 'label': 'Filter by name'},
               'nb_elements': {'value': nb_elements, 'type': 'int', 'label': 'Max number of elements to show'},
               }

    title = 'IT problems'
    if search:
        title = 'IT problems (%s)' % search

    return {'app': app, 'pbs': pbs, 'user': user, 'search': search, 'page': 'problems',
            'wid': wid, 'collapsed': collapsed, 'options': options, 'base_url': '/widget/problems', 'title': title,
            }

# Our page
def get_last_errors_widget():

    user = app.get_user_auth()
    if not user:
        app.bottle.redirect("/user/login")

    # We want to limit the number of elements, The user will be able to increase it
    nb_elements = max(0, int(app.request.GET.get('nb_elements', '10')))

    pbs = app.datamgr.get_problems_time_sorted()

    # Filter with the user interests
    pbs = only_related_to(pbs, user)

    # Keep only nb_elements
    pbs = pbs[:nb_elements]

    wid = app.request.GET.get('wid', 'widget_last_problems_' + str(int(time.time())))
    collapsed = (app.request.GET.get('collapsed', 'False') == 'True')

    options = {'nb_elements': {'value': nb_elements, 'type': 'int', 'label': 'Max number of elements to show'},
               }

    title = 'Last IT problems'

    return {'app': app, 'pbs': pbs, 'user': user, 'page': 'problems',
            'wid': wid, 'collapsed': collapsed, 'options': options, 'base_url': '/widget/last_problems', 'title': title,
            }

widget_desc = '''
IT problems

Show the most impacting IT problems
'''

last_widget_desc = '''
Last IT problems

Show the IT problems sorted by time
'''

pages = {
    get_page: {'routes': ['/problems'], 'view': 'problems', 'static': True},
    get_v1: {'routes': ['/problems-v1'], 'view': 'problems-v1', 'static': True},
    get_all: {'routes': ['/all'], 'view': 'problems', 'static': True},
    get_pbs_widget: {'routes': ['/widget/problems'], 'view': 'widget_problems', 'static': True, 'widget': ['dashboard'], 'widget_desc': widget_desc, 'widget_name': 'problems', 'widget_picture': '/static/problems/img/widget_problems.png'},
    get_last_errors_widget: {'routes': ['/widget/last_problems'], 'view': 'widget_last_problems', 'static': True, 'widget': ['dashboard'], 'widget_desc': last_widget_desc, 'widget_name': 'last_problems', 'widget_picture': '/static/problems/img/widget_problems.png'},
}
<commit_msg>Problems view - code cleaning<commit_after>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2014:
#    Gabes Jean, naparuba@gmail.com
#    Gerhard Lausser, Gerhard.Lausser@consol.de
#    Gregory Starck, g.starck@gmail.com
#    Hartmut Goebel, h.goebel@goebel-consult.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

from shinken.misc.filter  import only_related_to
from shinken.misc.sorter import hst_srv_sort
from shinken.objects.service import Service
from shinken.log import logger

### Will be populated by the UI with it's own value
app = None

import time
import re
import json

# Our page
def get_page():
    return get_view('problems')

# Our page
def get_all():
    return get_view('all')

# Our page
def get_v1():
    return get_view('problems')

# Our View code. We will get different data from all and /problems
# but it's mainly filtering changes
def get_view(page):

    user = app.get_user_auth()
    if not user:
        app.bottle.redirect("/user/login")

    # Look for the toolbar pref
    toolbar_pref = app.get_user_preference(user, 'toolbar')
    # If void, create an empty one
    if not toolbar_pref:
        app.set_user_preference(user, 'toolbar', 'show')
        toolbar_pref = 'show'
    toolbar = app.request.GET.get('toolbar', '')
    if toolbar != toolbar_pref and len(toolbar) > 0:
        print "Need to change user prefs for Toolbar",
        app.set_user_preference(user, 'toolbar', toolbar)
    toolbar_pref = app.get_user_preference(user, 'toolbar')

    # We want to limit the number of elements
    start = int(app.request.GET.get('start', '0'))
    end = int(app.request.GET.get('end', '30'))

    # We will keep a trace of our filters
    filters = {}
    ts = ['hst_srv', 'hg', 'realm', 'htag', 'stag', 'ack', 'downtime', 'crit']
    for t in ts:
        filters[t] = []

    search = app.request.GET.getall('search')
    if search == []:
        search = app.request.GET.get('global_search', '')

    # Most of the case, search will be a simple string, if so
    # make it a list of this string
    if isinstance(search, basestring):
        search = [search]

    search_str = '&'.join(search)

    # Load the bookmarks
    bookmarks_r = app.get_user_preference(user, 'bookmarks')
    if not bookmarks_r:
        app.set_user_preference(user, 'bookmarks', '[]')
        bookmarks_r = '[]'
    bookmarks = json.loads(bookmarks_r)
    bookmarks_ro = app.get_common_preference('bookmarks')
    if not bookmarks_ro:
        bookmarks_ro = '[]'

    bookmarksro = json.loads(bookmarks_ro)
    bookmarks = json.loads(bookmarks_r)

    items = []
    if page == 'problems':
        items = app.get_all_problems(user, to_sort=True, get_acknowledged=False)
    elif page == 'all':
        items = app.get_all_hosts_and_services(user)
    else:
        app.bottle.redirect("/problems")

    logger.debug("[%s] problems", app.name)
    for i in items:
        logger.debug("[%s] problems, item: %s", app.name, i.get_full_name())

    # Filter with the user interests
    # my_items = only_related_to(items, user)
    my_items = items

    # Check for related host contacts
    if not user.is_admin:
      for i in items:
        if isinstance(i,Service):
          if user in i.host.contacts:
            my_items.append(i)
            continue

    items = my_items
    logger.debug("[%s] problems after user filtering", app.name)
    for i in items:
        logger.debug("[%s] problems, item: %s", app.name, i.get_full_name())

    # Ok, if needed, apply the search filter
    for s in search:
        s = s.strip()
        if not s:
            continue

        logger.debug("[%s] problems, searching for: %s in %d items", app.name, s, len(items))

        elts = s.split(':', 1)
        t = 'hst_srv'
        if len(elts) > 1:
            t = elts[0]
            s = elts[1]

        logger.debug("[%s] problems, searching for type %s, pattern: %s", app.name, t, s)
        if not t in filters:
            filters[t] = []
        filters[t].append(s)

        if t == 'hst_srv':
            # We compile the pattern
            pat = re.compile(s, re.IGNORECASE)
            new_items = []
            for i in items:
                if pat.search(i.get_full_name()):
                    new_items.append(i)
                    continue
                to_add = False
                for imp in i.impacts:
                    if pat.search(imp.get_full_name()):
                        to_add = True
                for src in i.source_problems:
                    if pat.search(src.get_full_name()):
                        to_add = True
                if to_add:
                    new_items.append(i)

            items = new_items

        if t == 'hg':
            hg = app.datamgr.get_hostgroup(s)
            items = [i for i in items if hg in i.get_hostgroups()]

        if t == 'realm':
            r = app.datamgr.get_realm(s)
            items = [i for i in items if i.get_realm() == r]

        if t == 'htag':
            items = [i for i in items if s in i.get_host_tags()]

        if t == 'stag':
            items = [i for i in items if i.__class__.my_type == 'service' and s in i.get_service_tags()]

        if t == 'ack':
            if s == 'false':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or not i.problem_has_been_acknowledged]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (not i.problem_has_been_acknowledged and not i.host.problem_has_been_acknowledged)]
            if s == 'true':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or i.problem_has_been_acknowledged]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (i.problem_has_been_acknowledged or i.host.problem_has_been_acknowledged)]

        if t == 'downtime':
            if s == 'false':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or not i.in_scheduled_downtime]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (not i.in_scheduled_downtime and not i.host.in_scheduled_downtime)]
            if s == 'true':
                # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
                items = [i for i in items if i.__class__.my_type == 'service' or i.in_scheduled_downtime]
                # Now ok for hosts, but look for services, and service hosts
                items = [i for i in items if i.__class__.my_type == 'host' or (i.in_scheduled_downtime or i.host.in_scheduled_downtime)]

        if t == 'crit':
            items = [i for i in items if (i.__class__.my_type == 'service' and i.state_id == 2) or (i.__class__.my_type == 'host' and i.state_id == 1)]

        logger.debug("[%s] problems, found %d elements for type %s, pattern: %s", app.name, len(items), t, s)

    # If we are in the /problems and we do not have an ack filter
    # we apply by default the ack:false one
    if page == 'problems' and len(filters['ack']) == 0:
        # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
        items = [i for i in items if i.__class__.my_type == 'service' or not i.problem_has_been_acknowledged]
        # Now ok for hosts, but look for services, and service hosts
        items = [i for i in items if i.__class__.my_type == 'host' or (not i.problem_has_been_acknowledged and not i.host.problem_has_been_acknowledged)]

    # If we are in the /problems and we do not have an downtime filter
    # we apply by default the downtime:false one
    if page == 'problems' and len(filters['downtime']) == 0:
        # First look for hosts, so ok for services, but remove problem_has_been_acknowledged elements
        items = [i for i in items if i.__class__.my_type == 'service' or not i.in_scheduled_downtime]
        # Now ok for hosts, but look for services, and service hosts
        items = [i for i in items if i.__class__.my_type == 'host' or (not i.in_scheduled_downtime and not i.host.in_scheduled_downtime)]

    logger.debug("[%s] problems after search filtering", app.name)
    for i in items:
        logger.debug("[%s] problems, item: %s, %d, %d", app.name, i.get_full_name(), i.business_impact, i.state_id)

    # Now sort it!
    items.sort(hst_srv_sort)

    logger.debug("[%s] problems after host/service sort", app.name)
    for i in items:
        logger.debug("[%s] problems, item: %s, %d, %d", app.name, i.get_full_name(), i.business_impact, i.state_id)

    total = len(items)
    # If we overflow, came back as normal
    if start > total:
        start = 0
        end = 30

    navi = app.helper.get_navi(total, start, step=30)
    items = items[start:end]

    return {'app': app, 'pbs': items, 'user': user, 'navi': navi, 'search': search_str, 'page': page, 'filters': filters, 'bookmarks': bookmarks, 'bookmarksro': bookmarksro, 'toolbar': toolbar_pref }

# Our page
def get_pbs_widget():

    user = app.get_user_auth()
    if not user:
        app.bottle.redirect("/user/login")

    # We want to limit the number of elements, The user will be able to increase it
    nb_elements = max(0, int(app.request.GET.get('nb_elements', '10')))
    search = app.request.GET.get('search', '')

    pbs = app.datamgr.get_all_problems(to_sort=False)

    # Filter with the user interests
    pbs = only_related_to(pbs, user)

    # Sort it now
    pbs.sort(hst_srv_sort)

    # Ok, if need, appli the search filter
    if search:
        print "SEARCHING FOR", search
        print "Before filtering", len(pbs)
        # We compile the pattern
        pat = re.compile(search, re.IGNORECASE)
        new_pbs = []
        for p in pbs:
            if pat.search(p.get_full_name()):
                new_pbs.append(p)
                continue
            to_add = False
            for imp in p.impacts:
                if pat.search(imp.get_full_name()):
                    to_add = True
            for src in p.source_problems:
                if pat.search(src.get_full_name()):
                    to_add = True
            if to_add:
                new_pbs.append(p)

        pbs = new_pbs[:nb_elements]
        print "After filtering", len(pbs)

    pbs = pbs[:nb_elements]

    wid = app.request.GET.get('wid', 'widget_problems_' + str(int(time.time())))
    collapsed = (app.request.GET.get('collapsed', 'False') == 'True')

    options = {'search': {'value': search, 'type': 'text', 'label': 'Filter by name'},
               'nb_elements': {'value': nb_elements, 'type': 'int', 'label': 'Max number of elements to show'},
               }

    title = 'IT problems'
    if search:
        title = 'IT problems (%s)' % search

    return {'app': app, 'pbs': pbs, 'user': user, 'search': search, 'page': 'problems',
            'wid': wid, 'collapsed': collapsed, 'options': options, 'base_url': '/widget/problems', 'title': title,
            }

# Our page
def get_last_errors_widget():

    user = app.get_user_auth()
    if not user:
        app.bottle.redirect("/user/login")

    # We want to limit the number of elements, The user will be able to increase it
    nb_elements = max(0, int(app.request.GET.get('nb_elements', '10')))

    pbs = app.datamgr.get_problems_time_sorted()

    # Filter with the user interests
    pbs = only_related_to(pbs, user)

    # Keep only nb_elements
    pbs = pbs[:nb_elements]

    wid = app.request.GET.get('wid', 'widget_last_problems_' + str(int(time.time())))
    collapsed = (app.request.GET.get('collapsed', 'False') == 'True')

    options = {'nb_elements': {'value': nb_elements, 'type': 'int', 'label': 'Max number of elements to show'},
               }

    title = 'Last IT problems'

    return {'app': app, 'pbs': pbs, 'user': user, 'page': 'problems',
            'wid': wid, 'collapsed': collapsed, 'options': options, 'base_url': '/widget/last_problems', 'title': title,
            }

widget_desc = '''
IT problems

Show the most impacting IT problems
'''

last_widget_desc = '''
Last IT problems

Show the IT problems sorted by time
'''

pages = {
    get_page: {'routes': ['/problems'], 'view': 'problems', 'static': True},
    get_v1: {'routes': ['/problems-v1'], 'view': 'problems-v1', 'static': True},
    get_all: {'routes': ['/all'], 'view': 'problems', 'static': True},
    get_pbs_widget: {'routes': ['/widget/problems'], 'view': 'widget_problems', 'static': True, 'widget': ['dashboard'], 'widget_desc': widget_desc, 'widget_name': 'problems', 'widget_picture': '/static/problems/img/widget_problems.png'},
    get_last_errors_widget: {'routes': ['/widget/last_problems'], 'view': 'widget_last_problems', 'static': True, 'widget': ['dashboard'], 'widget_desc': last_widget_desc, 'widget_name': 'last_problems', 'widget_picture': '/static/problems/img/widget_problems.png'},
}
<|endoftext|>


<commit_before>from snakemake.exceptions import MissingInputException
import os
from os.path import join

_author__ = "Sebastian Kurscheid (sebastian.kurscheid@anu.edu.au)"
__license__ = "MIT"
__date__ = "2017-10-16"

rule:
    version: 0.1

localrules:
    all

# variables
home_dir = os.environ['HOME']
home = home_dir

wrapper_dir = os.environ['HOME'] + "/Development/snakemake-wrappers/bio"

REF_GENOME = "hg19"
REF_VERSION = config["references"][REF_GENOME]["version"][1]

# includes
include_prefix = os.environ['HOME'] + "/Development/JCSMR-Tremethick-Lab/HP1-alpha/snakemake/rules/"

# target files
BIGWIGs = expand("{assayID}/{file}.bw",
                 assayID="ChIP-Seq",
                 file=[i + "/" + config["processed_dir"] + "/" + REF_VERSION + "/deepTools/bamCoverage/normal/RPKM/duplicates_removed/"  + j + ".Q" + config["alignment_quality"]\
                    for i in config["samples"]["ChIP-Seq"]["runID"] \
                        for j in config["samples"]["ChIP-Seq"][i]])

# input functions
def get_sample_labels(wildcards):
    sl = []
    runIDs = config["samples"][wildcards["assayID"]]["runID"]
    for i in runIDs:
        for k in config["samples"][wildcards["assayID"]][i].keys():
            sl.append(k)
    return(sl)

def getSampleLabelsByCondition(wildcards):
    sl = []
    for i in config["samples"][wildcards["assayID"]]["conditions"][wildcards["condition"]]:
        for j in config["samples"][wildcards["assayID"]]["conditions"][wildcards["condition"]][i]:
            sl.append(j)
    return(sl)

def cli_parameters_computeMatrix(wildcards):
    if wildcards["command"] == "reference-point":
        a = config["program_parameters"][wildcards["application"]][wildcards["tool"]][wildcards["command"]]
        a["--referencePoint"]=wildcards["referencePoint"]
        return(a)
    if wildcards["command"] == "scale-regions":
        if wildcards["region"] in ["allGenes", "intergenicRegions"]:
            a = config["program_parameters"][wildcards["application"]][wildcards["tool"]][wildcards["command"]][wildcards["region"]]
            return(a)
        else:
            a = config["program_parameters"][wildcards["application"]][wildcards["tool"]][wildcards["command"]]["default"]
            return(a)

def cli_parameters_normalization(wildcards):
    if wildcards["norm"] == "RPKM":
        a = "--normalizeUsingRPKM"
    elif wildcards["norm"] == "1xcoverage":
        a = " ".join(("--normalizeTo1x", config["references"][REF_GENOME]["effectiveSize"]))
    return(a)

def getComputeMatrixInputMerged(wildcards):
    fn = []
    for i in config["samples"]["ChIP-Seq"]["replicates"].keys():
        fn.append("/".join(["ChIP-Seq",
                            "merged",
                            config["processed_dir"],
                            REF_VERSION,
                            "deepTools",
                            "bamCoverage",
                            wildcards["mode"],
                            wildcards["norm"],
                            wildcards["duplicates"],
                            i + ".bw"]))
    return(fn)

# subworkflows section
subworkflow mergeBams:
    workdir:  home + "/Data/Tremethick/HP1-alpha"
    snakefile: home + "Development/JCSMR-Tremethick-Lab/HP1-alpha/snakemake/workflows/subworkflows/mergeBam.py"

rule computeMatrix:
    version:
        0.3
    params:
        deepTools_dir = home + config["program_parameters"]["deepTools"]["deepTools_dir"],
        program_parameters = lambda wildcards: ' '.join("{!s}={!s}".format(key, val.strip("\\'")) for (key, val) in cli_parameters_computeMatrix(wildcards).items())
    threads:
        32
    input:
        file = getComputeMatrixInputMerged,
        region = lambda wildcards: home + config["program_parameters"]["deepTools"]["regionFiles"][wildcards["reference_version"]][wildcards["region"]]
    output:
        matrix_gz = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{region}_{mode}_{norm}.matrix.gz"
    shell:
        """
            {params.deepTools_dir}/computeMatrix {wildcards.command} \
                                                 --regionsFileName {input.region} \
                                                 --scoreFileName {input.file} \
                                                 --missingDataAsZero \
                                                 --skipZeros \
                                                 --numberOfProcessors {threads} \
                                                 {params.program_parameters} \
                                                 --outFileName {output.matrix_gz}
        """

rule plotProfileMerged:
    version:
        0.1
    params:
        deepTools_dir = home + config["program_parameters"]["deepTools"]["deepTools_dir"],
    input:
        matrix_gz = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/computeMatrix/{command}/{duplicates}/{referencePoint}/{region}_{mode}_{norm}.matrix.gz"
    output:
        figure = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.pdf",
        data = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.data",
        regions = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.bed"
    shell:
        """
            {params.deepTools_dir}/plotProfile --matrixFile {input.matrix_gz} \
                                               --outFileName {output.figure} \
                                               --outFileNameData {output.data} \
                                               --outFileSortedRegions {output.regions} \
                                               --plotType {wildcards.plotType}
        """

rule bigwigCompareMerged:
    version:
        0.1
    params:
        deepTools_dir = home + config["program_parameters"]["deepTools"]["deepTools_dir"],
    input:
        chip = lambda wildcards: "/".join(wildcards["assayID"],
                                           "merged",
                                           wildcards["outdir"],
                                           wildcards["reference_version"],
                                           wildcards["application"],
                                           "bamCoverage",
                                           wildcards["mode"],
                                           wildcards["norm"],
                                           wildcards["duplicates"],
                                           config["ChIP-Seq"]["ChIP-Input"][wildcards["condition"]]["ChIP"],
                                           ".bw"),
        input = lambda wildcards: "/".join(wildcards["assayID"],
                                           "merged",
                                           wildcards["outdir"],
                                           wildcards["reference_version"],
                                           wildcards["application"],
                                           "bamCoverage",
                                           wildcards["mode"],
                                           wildcards["norm"],
                                           wildcards["duplicates"],
                                           config["ChIP-Seq"]["ChIP-Input"][wildcards["condition"]]["Input"],
                                           ".bw")
    output:
        "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{mode}/{norm}/{duplicates}/{contrast}/{ratio}/{condition}.bw"
    shell:
        """
            {params.deepTools_dir}/bigwigCompare --bigwig1 {input.chip}\
                                                 --bigwig2 {input.input}\
                                                 --ratio {wildcards.ratio}\
                                                 --outFileFormat bigwig\
                                                 --outFileName {output}
        """

# target rules
rule all:
    input:
        expand("{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{mode}/{norm}/{duplicates}/{contrast}/{ratio}/{condition}.bw",
               assayID="ChIP-Seq",
               runID="merged",
               outdir=config["processed_dir"],
               reference_version=REF_VERSION,
               application=["deepTools"],
               tool=["plotProfile"],
               mode=["normal"],
               norm=["RPKM"],
               duplicates=["duplicates_removed"],
               contrast=["ChIP-Input"],
               ratio="log2",
               condition=["MCF10A_WT_HP1a",
                          "MCF10A_WT_HP1b",
                          "MCF10A_WT_H2AZ",
                          "MCF10A_shHP1a_HP1b",
                          "MCF10A_shHP1b_HP1a",
                          "MCF10A_shH2AZ_HP1a",
                          "MCF10A_shH2AZ_HP1b"]),
        expand("{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.{suffix}",
               assayID="ChIP-Seq",
               runID="merged",
               outdir=config["processed_dir"],
               reference_version=REF_VERSION,
               application="deepTools",
               tool="plotProfile",
               command=["scale-regions"],
               duplicates=["duplicates_removed"],
               referencePoint="TSS",
               plotType="se",
               mode=["normal"],
               norm=["RPKM"],
               region=["allGenes", "intergenicRegions", "conditionMCF10A_shH2AZHP1a", "conditionMCF10A_shHP1ab", "conditionMCF10A_shHP1a", "conditionMCF10A_shHP1b", "conditionMCF10A_WT"],
               suffix=["pdf", "data", "bed"]),
<commit_msg>attempts to debug wildcards<commit_after>from snakemake.exceptions import MissingInputException
import os
from os.path import join

_author__ = "Sebastian Kurscheid (sebastian.kurscheid@anu.edu.au)"
__license__ = "MIT"
__date__ = "2017-10-16"

rule:
    version: 0.1

localrules:
    all

# variables
home_dir = os.environ['HOME']
home = home_dir

wrapper_dir = os.environ['HOME'] + "/Development/snakemake-wrappers/bio"

REF_GENOME = "hg19"
REF_VERSION = config["references"][REF_GENOME]["version"][1]

# includes
include_prefix = os.environ['HOME'] + "/Development/JCSMR-Tremethick-Lab/HP1-alpha/snakemake/rules/"

# target files
BIGWIGs = expand("{assayID}/{file}.bw",
                 assayID="ChIP-Seq",
                 file=[i + "/" + config["processed_dir"] + "/" + REF_VERSION + "/deepTools/bamCoverage/normal/RPKM/duplicates_removed/"  + j + ".Q" + config["alignment_quality"]\
                    for i in config["samples"]["ChIP-Seq"]["runID"] \
                        for j in config["samples"]["ChIP-Seq"][i]])

# input functions
def get_sample_labels(wildcards):
    sl = []
    runIDs = config["samples"][wildcards["assayID"]]["runID"]
    for i in runIDs:
        for k in config["samples"][wildcards["assayID"]][i].keys():
            sl.append(k)
    return(sl)

def getSampleLabelsByCondition(wildcards):
    sl = []
    for i in config["samples"][wildcards["assayID"]]["conditions"][wildcards["condition"]]:
        for j in config["samples"][wildcards["assayID"]]["conditions"][wildcards["condition"]][i]:
            sl.append(j)
    return(sl)

def cli_parameters_computeMatrix(wildcards):
    if wildcards["command"] == "reference-point":
        a = config["program_parameters"][wildcards["application"]][wildcards["tool"]][wildcards["command"]]
        a["--referencePoint"]=wildcards["referencePoint"]
        return(a)
    if wildcards["command"] == "scale-regions":
        if wildcards["region"] in ["allGenes", "intergenicRegions"]:
            a = config["program_parameters"][wildcards["application"]][wildcards["tool"]][wildcards["command"]][wildcards["region"]]
            return(a)
        else:
            a = config["program_parameters"][wildcards["application"]][wildcards["tool"]][wildcards["command"]]["default"]
            return(a)

def cli_parameters_normalization(wildcards):
    if wildcards["norm"] == "RPKM":
        a = "--normalizeUsingRPKM"
    elif wildcards["norm"] == "1xcoverage":
        a = " ".join(("--normalizeTo1x", config["references"][REF_GENOME]["effectiveSize"]))
    return(a)

def getComputeMatrixInputMerged(wildcards):
    fn = []
    for i in config["samples"]["ChIP-Seq"]["replicates"].keys():
        fn.append("/".join(["ChIP-Seq",
                            "merged",
                            config["processed_dir"],
                            REF_VERSION,
                            "deepTools",
                            "bamCoverage",
                            wildcards["mode"],
                            wildcards["norm"],
                            wildcards["duplicates"],
                            i + ".bw"]))
    return(fn)

def debugWildcards(wildcards):
    print(wildcards)

# subworkflows section
subworkflow mergeBams:
    workdir:  home + "/Data/Tremethick/HP1-alpha"
    snakefile: home + "Development/JCSMR-Tremethick-Lab/HP1-alpha/snakemake/workflows/subworkflows/mergeBam.py"

rule computeMatrix:
    version:
        0.3
    params:
        deepTools_dir = home + config["program_parameters"]["deepTools"]["deepTools_dir"],
        program_parameters = lambda wildcards: ' '.join("{!s}={!s}".format(key, val.strip("\\'")) for (key, val) in cli_parameters_computeMatrix(wildcards).items())
    threads:
        32
    input:
        file = getComputeMatrixInputMerged,
        region = lambda wildcards: home + config["program_parameters"]["deepTools"]["regionFiles"][wildcards["reference_version"]][wildcards["region"]]
    output:
        matrix_gz = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{region}_{mode}_{norm}.matrix.gz"
    shell:
        """
            {params.deepTools_dir}/computeMatrix {wildcards.command} \
                                                 --regionsFileName {input.region} \
                                                 --scoreFileName {input.file} \
                                                 --missingDataAsZero \
                                                 --skipZeros \
                                                 --numberOfProcessors {threads} \
                                                 {params.program_parameters} \
                                                 --outFileName {output.matrix_gz}
        """

rule plotProfileMerged:
    version:
        0.1
    params:
        deepTools_dir = home + config["program_parameters"]["deepTools"]["deepTools_dir"],
    input:
        matrix_gz = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/computeMatrix/{command}/{duplicates}/{referencePoint}/{region}_{mode}_{norm}.matrix.gz"
    output:
        figure = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.pdf",
        data = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.data",
        regions = "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.bed"
    shell:
        """
            {params.deepTools_dir}/plotProfile --matrixFile {input.matrix_gz} \
                                               --outFileName {output.figure} \
                                               --outFileNameData {output.data} \
                                               --outFileSortedRegions {output.regions} \
                                               --plotType {wildcards.plotType}
        """

rule bigwigCompareMerged:
    version:
        0.1
    params:
        deepTools_dir = home + config["program_parameters"]["deepTools"]["deepTools_dir"],
        debug = debugWildcards
    input:
        chip = lambda wildcards: "/".join(wildcards["assayID"],
                                           "merged",
                                           wildcards["outdir"],
                                           wildcards["reference_version"],
                                           wildcards["application"],
                                           "bamCoverage",
                                           wildcards["mode"],
                                           wildcards["norm"],
                                           wildcards["duplicates"],
                                           config["ChIP-Seq"]["ChIP-Input"][wildcards["condition"]]["ChIP"],
                                           ".bw"),
        input = lambda wildcards: "/".join(wildcards["assayID"],
                                           "merged",
                                           wildcards["outdir"],
                                           wildcards["reference_version"],
                                           wildcards["application"],
                                           "bamCoverage",
                                           wildcards["mode"],
                                           wildcards["norm"],
                                           wildcards["duplicates"],
                                           config["ChIP-Seq"]["ChIP-Input"][wildcards["condition"]]["Input"],
                                           ".bw")
    output:
        "{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{mode}/{norm}/{duplicates}/{contrast}/{ratio}/{condition}.bw"
    shell:
        """
            {params.deepTools_dir}/bigwigCompare --bigwig1 {input.chip}\
                                                 --bigwig2 {input.input}\
                                                 --ratio {wildcards.ratio}\
                                                 --outFileFormat bigwig\
                                                 --outFileName {output}
        """

# target rules
rule all:
    input:
        expand("{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{mode}/{norm}/{duplicates}/{contrast}/{ratio}/{condition}.bw",
               assayID="ChIP-Seq",
               runID="merged",
               outdir=config["processed_dir"],
               reference_version=REF_VERSION,
               application=["deepTools"],
               tool=["plotProfile"],
               mode=["normal"],
               norm=["RPKM"],
               duplicates=["duplicates_removed"],
               contrast=["ChIP-Input"],
               ratio="log2",
               condition=["MCF10A_WT_HP1a",
                          "MCF10A_WT_HP1b",
                          "MCF10A_WT_H2AZ",
                          "MCF10A_shHP1a_HP1b",
                          "MCF10A_shHP1b_HP1a",
                          "MCF10A_shH2AZ_HP1a",
                          "MCF10A_shH2AZ_HP1b"]),
        expand("{assayID}/{runID}/{outdir}/{reference_version}/{application}/{tool}/{command}/{duplicates}/{referencePoint}/{plotType}.{mode}.{norm}.{region}.{suffix}",
               assayID="ChIP-Seq",
               runID="merged",
               outdir=config["processed_dir"],
               reference_version=REF_VERSION,
               application="deepTools",
               tool="plotProfile",
               command=["scale-regions"],
               duplicates=["duplicates_removed"],
               referencePoint="TSS",
               plotType="se",
               mode=["normal"],
               norm=["RPKM"],
               region=["allGenes", "intergenicRegions", "conditionMCF10A_shH2AZHP1a", "conditionMCF10A_shHP1ab", "conditionMCF10A_shHP1a", "conditionMCF10A_shHP1b", "conditionMCF10A_WT"],
               suffix=["pdf", "data", "bed"]),
<|endoftext|>


<commit_before>import sys
import logging

def launch():
    logging.basicConfig(stream=sys.stderr)
    from protvr import app as application
    # TODO: move this to an environment variable or somesuch
    application.secret_key = 'vnmytyuyerwfsdfbyhfghjytrer'
<commit_msg>Delete launch.py<commit_after><|endoftext|>


<commit_before>from heuristic import Heuristic

class Stalkernet_Heuristic(Heuristic):
    def __init__(self):
        self.input_list = []
        self.read_input_file()
        super(Stalkernet_Heuristic, self).__init__(self.input_list)

    # Returns a heuristic value for a list of keywords
    def run_heuristic(self, keywords):
        toBeReturned = 0
        for word in keywords:
            if word in self.input_list:
                toBeReturned += self.input_list[word]
        return toBeReturned

    def read_input_file(self):
        # read input file into dictionary with keyword as key and
        # heuristic score as value
        input_file = open('stalkernet_values.txt', 'rb')
        for line in input_file:
            cur_line = line.split(',')
            self.input_list += [(cur_line[0], int(cur_line[1]))]

if __name__ == '__main__':
    my_heuristic = Stalkernet_Heuristic()
    print my_heuristic.list_of_values.get_magnitudes()
<commit_msg>Refactored to take input of keywords and values from txt file<commit_after>from heuristic import Heuristic

class StalkernetHeuristic(Heuristic):
    def __init__(self):
        self.input_list = []
        self.read_input_file()
        super(StalkernetHeuristic, self).__init__(self.input_list)

    # Returns a heuristic value for a list of keywords
    def run_heuristic(self, keywords):
        toBeReturned = 0
        for word in keywords:
            if word in self.input_list:
                toBeReturned += self.input_list[word]
        return toBeReturned

    def read_input_file(self):
        # read input file into dictionary with keyword as key and
        # heuristic score as value
        input_file = open('stalkernet_values.txt', 'rb')
        for line in input_file:
            cur_line = line.split(',')
            self.input_list += [(cur_line[0], int(cur_line[1]))]

if __name__ == '__main__':
    my_heuristic = StalkernetHeuristic()
    print my_heuristic.list_of_values.get_magnitudes()
<|endoftext|>