<commit_before>from ..base import Module, command, master_only, EMOTES

from .errors import *
from .player import Player
from .queue import SongQueue, RandomQueue
from .queue_display import QueueDisplay
from .song import Song

from . import constants as c
from . import tools as t

class Dj(Module):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.invoked_channel = None
        self.voice_channel = None
        self.voice = None

        self.queue = SongQueue()
        self.backup_queue = RandomQueue()
        self.player = Player(self, self.queue, self.backup_queue)

        self.queue_display = QueueDisplay(self)

    def on_message(self, message):
        if message.channel == self.invoked_channel:
            self.queue_display.pull_down()

    @command('!dj join {channel_name}', master_only)
    async def join_room(self, message, channel_name):
        if self.invoked_channel:
            raise AlreadyInvoked(self.invoked_channel)
        # Finding channel by name in the current server
        voice_channel = self.bot.find_voice_channel(
            channel_name,
            message.server
        )
        if voice_channel is None:
            raise NonExistentChannel(channel_name)
        # Attempting to join
        try:
            voice = await self.bot.join_voice_channel(voice_channel)
        except Exception as e:
            raise VoiceChannelJoinError(e)
        # Success
        self.invoked_channel = message.channel
        self.voice_channel = voice_channel
        self.voice = voice
        # Start playing
        self.run_async(
            self.player.run_forever(),
            self.invoked_channel
        )
        self.player.play()
        # Enable queue display
        self.run_async(
            self.queue_display.run(),
            self.invoked_channel
        )
        # Notifying success
        await self.send_message(
            message.channel,
            (
                'Ready to play music in `{}`!\n'
                'Use the `!dj play youtube_id_or_link` in this channel '
                'to queue songs {}'
            ).format(
                self.voice_channel.name,
                EMOTES.LirikGood
            )
        )

    @command('!dj retire', master_only)
    async def retire(self, message):
        self.ensure_channel(message)
        self.player.stop()
        # Attempting to disconnect
        try:
            await self.voice.disconnect()
        except Exception as e:
            raise VoiceChannelLeaveError(e)
        # Success
        self.invoked_channel = None
        self.voice_channel = None
        self.voice = None

        self.queue.clear()

        await self.send_message(
            message.channel,
            'Bye'
        )

    @command('!dj queue')
    async def toggle_queue(self, message):
        self.ensure_channel(message)
        # Toggling the queue display
        self.queue_display.toggle()

    @command('!dj play {song_link}')
    async def play_song(self, message, song_link):
        self.ensure_channel(message)
        # Queue limit
        if len(self.queue.user_songs(message.author.id)) > c.MAX_VIDEO_PER_USER:
            raise TooManyQueuedForUser
        song = Song(song_link, message)
        # Song duration
        if song.duration >= c.MAX_VIDEO_DURATION:
            raise VideoTooLong
        # Uniqueness
        if song in self.queue or song == self.player.current_song:
            raise AlreadyQueued

        await self.send_message(
            message.channel,
            (
                '{} Your song: {} has been queued {}\n'
                'It will be played in `{}`'
            ).format(
                message.author.mention,
                song.formatted(),
                EMOTES.LirikChamp,
                t.format_seconds(self.player.queue_duration)
            ),
            c.CLEAR_INTERVAL
        )
        # Queuing it
        self.queue.append(song)

    @command('!dj skip')
    async def skip_song(self, message):
        self.ensure_channel(message)
        self.ensure_listening(message)
        self.player.skip()

    @command('!dj blacklist')
    async def blacklist_song(self, message):
        raise NotImplemented
        # self.ensure_channel(message)
        # self.ensure_listening(message)

    def ensure_channel(self, message):
        if self.invoked_channel is None:
            raise NotInvoked
        if self.invoked_channel != message.channel:
            raise WrongChannel(message.channel)

    def ensure_listening(self, message):
        pass

    async def play_error(self, song, exception):
        await self.send_message(
            self.invoked_channel,
            (
                '{} I was unable to play your song {}\n'
                '```\n'
                '{}\n'
                '```'
            ).format(
                song.message.author.mention,
                EMOTES.LirikFeels,
                exception
            )
        )
<commit_msg>Set the skip command to master only for now<commit_after>from ..base import Module, command, master_only, EMOTES

from .errors import *
from .player import Player
from .queue import SongQueue, RandomQueue
from .queue_display import QueueDisplay
from .song import Song

from . import constants as c
from . import tools as t

class Dj(Module):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.invoked_channel = None
        self.voice_channel = None
        self.voice = None

        self.queue = SongQueue()
        self.backup_queue = RandomQueue()
        self.player = Player(self, self.queue, self.backup_queue)

        self.queue_display = QueueDisplay(self)

    def on_message(self, message):
        if message.channel == self.invoked_channel:
            self.queue_display.pull_down()

    @command('!dj join {channel_name}', master_only)
    async def join_room(self, message, channel_name):
        if self.invoked_channel:
            raise AlreadyInvoked(self.invoked_channel)
        # Finding channel by name in the current server
        voice_channel = self.bot.find_voice_channel(
            channel_name,
            message.server
        )
        if voice_channel is None:
            raise NonExistentChannel(channel_name)
        # Attempting to join
        try:
            voice = await self.bot.join_voice_channel(voice_channel)
        except Exception as e:
            raise VoiceChannelJoinError(e)
        # Success
        self.invoked_channel = message.channel
        self.voice_channel = voice_channel
        self.voice = voice
        # Start playing
        self.run_async(
            self.player.run_forever(),
            self.invoked_channel
        )
        self.player.play()
        # Enable queue display
        self.run_async(
            self.queue_display.run(),
            self.invoked_channel
        )
        # Notifying success
        await self.send_message(
            message.channel,
            (
                'Ready to play music in `{}`!\n'
                'Use the `!dj play youtube_id_or_link` in this channel '
                'to queue songs {}'
            ).format(
                self.voice_channel.name,
                EMOTES.LirikGood
            )
        )

    @command('!dj retire', master_only)
    async def retire(self, message):
        self.ensure_channel(message)
        self.player.stop()
        # Attempting to disconnect
        try:
            await self.voice.disconnect()
        except Exception as e:
            raise VoiceChannelLeaveError(e)
        # Success
        self.invoked_channel = None
        self.voice_channel = None
        self.voice = None

        self.queue.clear()

        await self.send_message(
            message.channel,
            'Bye'
        )

    @command('!dj queue')
    async def toggle_queue(self, message):
        self.ensure_channel(message)
        # Toggling the queue display
        self.queue_display.toggle()

    @command('!dj play {song_link}')
    async def play_song(self, message, song_link):
        self.ensure_channel(message)
        # Queue limit
        if len(self.queue.user_songs(message.author.id)) > c.MAX_VIDEO_PER_USER:
            raise TooManyQueuedForUser
        song = Song(song_link, message)
        # Song duration
        if song.duration >= c.MAX_VIDEO_DURATION:
            raise VideoTooLong
        # Uniqueness
        if song in self.queue or song == self.player.current_song:
            raise AlreadyQueued

        await self.send_message(
            message.channel,
            (
                '{} Your song: {} has been queued {}\n'
                'It will be played in `{}`'
            ).format(
                message.author.mention,
                song.formatted(),
                EMOTES.LirikChamp,
                t.format_seconds(self.player.queue_duration)
            ),
            c.CLEAR_INTERVAL
        )
        # Queuing it
        self.queue.append(song)

    @command('!dj skip', master_only)
    async def skip_song(self, message):
        self.ensure_channel(message)
        self.ensure_listening(message)
        self.player.skip()

    @command('!dj blacklist')
    async def blacklist_song(self, message):
        raise NotImplemented
        # self.ensure_channel(message)
        # self.ensure_listening(message)

    def ensure_channel(self, message):
        if self.invoked_channel is None:
            raise NotInvoked
        if self.invoked_channel != message.channel:
            raise WrongChannel(message.channel)

    def ensure_listening(self, message):
        pass

    async def play_error(self, song, exception):
        await self.send_message(
            self.invoked_channel,
            (
                '{} I was unable to play your song {}\n'
                '```\n'
                '{}\n'
                '```'
            ).format(
                song.message.author.mention,
                EMOTES.LirikFeels,
                exception
            )
        )
<|endoftext|>


<commit_before>"""Identify program versions used for analysis, reporting in structured table.

Catalogs the full list of programs used in analysis, enabling reproduction of
results and tracking of provenance in output files.
"""
import os
import contextlib
import subprocess
import sys

import yaml

from bcbio import utils
from bcbio.pipeline import config_utils, version
from bcbio.log import logger

_cl_progs = [{"cmd": "bamtofastq", "name": "biobambam",
              "args": "--version", "stdout_flag": "This is biobambam version"},
             {"cmd": "bamtools", "args": "--version", "stdout_flag": "bamtools"},
             {"cmd": "bcftools", "stdout_flag": "Version:"},
             {"cmd": "bedtools", "args": "--version", "stdout_flag": "bedtools"},
             {"cmd": "bowtie2", "args": "--version", "stdout_flag": "bowtie2-align version"},
             {"cmd": "bwa", "stdout_flag": "Version:"},
             {"cmd": "cufflinks", "stdout_flag": "cufflinks"},
             {"cmd": "cutadapt", "args": "--version"},
             {"cmd": "fastqc", "args": "--version", "stdout_flag": "FastQC"},
             {"cmd": "freebayes", "stdout_flag": "version:"},
             {"cmd": "gemini", "args": "--version", "stdout_flag": "gemini "},
             {"cmd": "novosort", "paren_flag": "novosort"},
             {"cmd": "novoalign", "stdout_flag": "Novoalign"},
             {"cmd": "samtools", "stdout_flag": "Version:"},
             {"cmd": "sambamba", "stdout_flag": "sambamba"},
             {"cmd": "qualimap", "args": "-h", "stdout_flag": "QualiMap"},
             {"cmd": "tophat", "args": "--version", "stdout_flag": "TopHat"},
             {"cmd": "vcflib", "has_cl_version": False},
             {"cmd": "featurecounts", "args": "-v", "stdout_flag": "featureCounts"}]

def _broad_versioner(type):
    def get_version(config):
        from bcbio import broad
        try:
            runner = broad.runner_from_config(config)
        except ValueError:
            return ""
        if type == "gatk":
            return runner.get_gatk_version()
        elif type == "picard":
            return runner.get_picard_version("ViewSam")
        elif type == "mutect":
            try:
                runner = broad.runner_from_config(config, "mutect")
            except ValueError:
                return ""
            return runner.get_mutect_version()
        else:
            raise NotImplementedError(type)
    return get_version

def jar_versioner(program_name, jar_name):
    """Retrieve version information based on jar file.
    """
    def get_version(config):
        try:
            pdir = config_utils.get_program(program_name, config, "dir")
        # not configured
        except ValueError:
            return ""
        jar = os.path.basename(config_utils.get_jar(jar_name, pdir))
        for to_remove in [jar_name, ".jar", "-standalone"]:
            jar = jar.replace(to_remove, "")
        if jar.startswith(("-", ".")):
            jar = jar[1:]
        if jar is "":
            logger.warn("Unable to determine version for program '{}' from jar file {}".format(
                program_name, config_utils.get_jar(jar_name, pdir)))
        return jar
    return get_version

def java_versioner(pname, jar_name, **kwargs):
    def get_version(config):
        try:
            pdir = config_utils.get_program(pname, config, "dir")
        except ValueError:
            return ""
        jar = config_utils.get_jar(jar_name, pdir)
        kwargs["cmd"] = "java"
        kwargs["args"] = "-Xms128m -Xmx256m -jar %s" % jar
        return _get_cl_version(kwargs, config)
    return get_version

_alt_progs = [{"name": "bcbio_variation",
               "version_fn": jar_versioner("bcbio_variation", "bcbio.variation")},
              {"name": "gatk", "version_fn": _broad_versioner("gatk")},
              {"name": "mutect",
               "version_fn": _broad_versioner("mutect")},
              {"name": "picard", "version_fn": _broad_versioner("picard")},
              {"name": "rnaseqc",
               "version_fn": jar_versioner("rnaseqc", "RNA-SeQC")},
              {"name": "snpeff",
               "version_fn": java_versioner("snpeff", "snpEff", stdout_flag="snpEff version SnpEff")},
              {"name": "varscan",
               "version_fn": jar_versioner("varscan", "VarScan")},
              {"name": "oncofuse",
               "version_fn": jar_versioner("Oncofuse", "Oncofuse")},
              {"name": "alientrimmer",
               "version_fn": jar_versioner("AlienTrimmer", "AlienTrimmer")}
]

def _parse_from_stdoutflag(stdout, x):
    for line in stdout:
        if line.find(x) >= 0:
            parts = [p for p in line[line.find(x) + len(x):].split() if p.strip()]
            return parts[0].strip()
    return ""

def _parse_from_parenflag(stdout, x):
    for line in stdout:
        if line.find(x) >= 0:
            return line.split("(")[-1].split(")")[0]
    return ""

def _get_cl_version(p, config):
    """Retrieve version of a single commandline program.
    """
    if not p.get("has_cl_version", True):
        return ""
    try:
        prog = config_utils.get_program(p["cmd"], config)
    except config_utils.CmdNotFound:
        localpy_cmd = os.path.join(os.path.dirname(sys.executable), p["cmd"])
        if os.path.exists(localpy_cmd):
            prog = localpy_cmd
        else:
            return ""
    args = p.get("args", "")

    cmd = "{prog} {args}"
    subp = subprocess.Popen(cmd.format(**locals()), stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT,
                            shell=True)
    with contextlib.closing(subp.stdout) as stdout:
        if p.get("stdout_flag"):
            v = _parse_from_stdoutflag(stdout, p["stdout_flag"])
        elif p.get("paren_flag"):
            v = _parse_from_parenflag(stdout, p["paren_flag"])
        else:
            v = stdout.read().strip()
    if v.endswith("."):
        v = v[:-1]
    return v

def _get_brew_versions():
    """Retrieve versions of tools installed via brew.
    """
    from bcbio import install
    tooldir = install.get_defaults().get("tooldir")
    brew_cmd = os.path.join(tooldir, "bin", "brew") if tooldir else "brew"
    try:
        vout = subprocess.check_output([brew_cmd, "which"])
        uses_which = True
    except subprocess.CalledProcessError:
        vout = subprocess.check_output([brew_cmd, "list", "--versions"])
        uses_which = False
    except OSError:  # brew not installed/used
        vout = ""
    out = {}
    for vstr in vout.split("\n"):
        if vstr.strip():
            if uses_which:
                name, v = vstr.rstrip().split(": ")
            else:
                parts = vstr.rstrip().split()
                name = parts[0]
                v = parts[-1]
            out[name] = v
    return out

def _get_versions(config=None):
    """Retrieve details on all programs available on the system.
    """
    out = [{"program": "bcbio-nextgen",
            "version": ("%s-%s" % (version.__version__, version.__git_revision__)
                        if version.__git_revision__ else version.__version__)}]
    manifest_vs = _get_versions_manifest()
    if manifest_vs:
        return out + manifest_vs
    else:
        assert config is not None, "Need configuration to retrieve from non-manifest installs"
        brew_vs = _get_brew_versions()
        import HTSeq
        out.append({"program": "htseq", "version": HTSeq.__version__})
        for p in _cl_progs:
            out.append({"program": p["cmd"],
                        "version": (brew_vs[p["cmd"]] if p["cmd"] in brew_vs else
                                    _get_cl_version(p, config))})
        for p in _alt_progs:
            out.append({"program": p["name"],
                        "version": (brew_vs[p["name"]] if p["name"] in brew_vs else
                                    p["version_fn"](config))})
        return out

def _get_versions_manifest():
    """Retrieve versions from a pre-existing manifest of installed software.
    """
    all_pkgs = ["htseq", "cn.mops", "vt", "platypus-variant", "gatk-framework"] + \
               [p.get("name", p["cmd"]) for p in _cl_progs] + [p["name"] for p in _alt_progs]
    manifest_dir = os.path.join(config_utils.get_base_installdir(), "manifest")
    if os.path.exists(manifest_dir):
        out = []
        for plist in ["toolplus", "brew", "python", "r", "debian", "custom"]:
            pkg_file = os.path.join(manifest_dir, "%s-packages.yaml" % plist)
            if os.path.exists(pkg_file):
                with open(pkg_file) as in_handle:
                    pkg_info = yaml.safe_load(in_handle)
                added = []
                for pkg in all_pkgs:
                    if pkg in pkg_info:
                        added.append(pkg)
                        out.append({"program": pkg, "version": pkg_info[pkg]["version"]})
                for x in added:
                    all_pkgs.remove(x)
        out.sort(key=lambda x: x["program"])
        for pkg in all_pkgs:
            out.append({"program": pkg, "version": ""})
        return out

def _get_program_file(dirs):
    if dirs.get("work"):
        base_dir = utils.safe_makedir(os.path.join(dirs["work"], "provenance"))
        return os.path.join(base_dir, "programs.txt")

def write_versions(dirs, config=None, is_wrapper=False):
    """Write CSV file with versions used in analysis pipeline.
    """
    out_file = _get_program_file(dirs)
    if is_wrapper:
        assert utils.file_exists(out_file), "Failed to create program versions from VM"
    elif out_file is None:
        for p in _get_versions(config):
            print("{program},{version}".format(**p))
    else:
        with open(out_file, "w") as out_handle:
            for p in _get_versions(config):
                out_handle.write("{program},{version}\n".format(**p))
    return out_file

def add_subparser(subparsers):
    """Add command line option for exporting version information.
    """
    parser = subparsers.add_parser("version",
                                   help="Export versions of used software to stdout or a file ")
    parser.add_argument("--workdir", help="Directory export programs to in workdir/provenance/programs.txt",
                        default=None)

def get_version(name, dirs=None, config=None):
    """Retrieve the current version of the given program from cached names.
    """
    if dirs:
        p = _get_program_file(dirs)
    else:
        p = config["resources"]["program_versions"]
    with open(p) as in_handle:
        for line in in_handle:
            prog, version = line.rstrip().split(",")
            if prog == name and version:
                return version
    raise KeyError("Version information not found for %s in %s" % (name, p))
<commit_msg>Avoid including Python warnings in version output. Fixes #397<commit_after>"""Identify program versions used for analysis, reporting in structured table.

Catalogs the full list of programs used in analysis, enabling reproduction of
results and tracking of provenance in output files.
"""
import os
import contextlib
import subprocess
import sys

import yaml

from bcbio import utils
from bcbio.pipeline import config_utils, version
from bcbio.log import logger

_cl_progs = [{"cmd": "bamtofastq", "name": "biobambam",
              "args": "--version", "stdout_flag": "This is biobambam version"},
             {"cmd": "bamtools", "args": "--version", "stdout_flag": "bamtools"},
             {"cmd": "bcftools", "stdout_flag": "Version:"},
             {"cmd": "bedtools", "args": "--version", "stdout_flag": "bedtools"},
             {"cmd": "bowtie2", "args": "--version", "stdout_flag": "bowtie2-align version"},
             {"cmd": "bwa", "stdout_flag": "Version:"},
             {"cmd": "cufflinks", "stdout_flag": "cufflinks"},
             {"cmd": "cutadapt", "args": "--version"},
             {"cmd": "fastqc", "args": "--version", "stdout_flag": "FastQC"},
             {"cmd": "freebayes", "stdout_flag": "version:"},
             {"cmd": "gemini", "args": "--version", "stdout_flag": "gemini "},
             {"cmd": "novosort", "paren_flag": "novosort"},
             {"cmd": "novoalign", "stdout_flag": "Novoalign"},
             {"cmd": "samtools", "stdout_flag": "Version:"},
             {"cmd": "sambamba", "stdout_flag": "sambamba"},
             {"cmd": "qualimap", "args": "-h", "stdout_flag": "QualiMap"},
             {"cmd": "tophat", "args": "--version", "stdout_flag": "TopHat"},
             {"cmd": "vcflib", "has_cl_version": False},
             {"cmd": "featurecounts", "args": "-v", "stdout_flag": "featureCounts"}]

def _broad_versioner(type):
    def get_version(config):
        from bcbio import broad
        try:
            runner = broad.runner_from_config(config)
        except ValueError:
            return ""
        if type == "gatk":
            return runner.get_gatk_version()
        elif type == "picard":
            return runner.get_picard_version("ViewSam")
        elif type == "mutect":
            try:
                runner = broad.runner_from_config(config, "mutect")
            except ValueError:
                return ""
            return runner.get_mutect_version()
        else:
            raise NotImplementedError(type)
    return get_version

def jar_versioner(program_name, jar_name):
    """Retrieve version information based on jar file.
    """
    def get_version(config):
        try:
            pdir = config_utils.get_program(program_name, config, "dir")
        # not configured
        except ValueError:
            return ""
        jar = os.path.basename(config_utils.get_jar(jar_name, pdir))
        for to_remove in [jar_name, ".jar", "-standalone"]:
            jar = jar.replace(to_remove, "")
        if jar.startswith(("-", ".")):
            jar = jar[1:]
        if jar is "":
            logger.warn("Unable to determine version for program '{}' from jar file {}".format(
                program_name, config_utils.get_jar(jar_name, pdir)))
        return jar
    return get_version

def java_versioner(pname, jar_name, **kwargs):
    def get_version(config):
        try:
            pdir = config_utils.get_program(pname, config, "dir")
        except ValueError:
            return ""
        jar = config_utils.get_jar(jar_name, pdir)
        kwargs["cmd"] = "java"
        kwargs["args"] = "-Xms128m -Xmx256m -jar %s" % jar
        return _get_cl_version(kwargs, config)
    return get_version

_alt_progs = [{"name": "bcbio_variation",
               "version_fn": jar_versioner("bcbio_variation", "bcbio.variation")},
              {"name": "gatk", "version_fn": _broad_versioner("gatk")},
              {"name": "mutect",
               "version_fn": _broad_versioner("mutect")},
              {"name": "picard", "version_fn": _broad_versioner("picard")},
              {"name": "rnaseqc",
               "version_fn": jar_versioner("rnaseqc", "RNA-SeQC")},
              {"name": "snpeff",
               "version_fn": java_versioner("snpeff", "snpEff", stdout_flag="snpEff version SnpEff")},
              {"name": "varscan",
               "version_fn": jar_versioner("varscan", "VarScan")},
              {"name": "oncofuse",
               "version_fn": jar_versioner("Oncofuse", "Oncofuse")},
              {"name": "alientrimmer",
               "version_fn": jar_versioner("AlienTrimmer", "AlienTrimmer")}
]

def _parse_from_stdoutflag(stdout, x):
    for line in stdout:
        if line.find(x) >= 0:
            parts = [p for p in line[line.find(x) + len(x):].split() if p.strip()]
            return parts[0].strip()
    return ""

def _parse_from_parenflag(stdout, x):
    for line in stdout:
        if line.find(x) >= 0:
            return line.split("(")[-1].split(")")[0]
    return ""

def _get_cl_version(p, config):
    """Retrieve version of a single commandline program.
    """
    if not p.get("has_cl_version", True):
        return ""
    try:
        prog = config_utils.get_program(p["cmd"], config)
    except config_utils.CmdNotFound:
        localpy_cmd = os.path.join(os.path.dirname(sys.executable), p["cmd"])
        if os.path.exists(localpy_cmd):
            prog = localpy_cmd
        else:
            return ""
    args = p.get("args", "")

    cmd = "{prog} {args}"
    subp = subprocess.Popen(cmd.format(**locals()), stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT,
                            shell=True)
    with contextlib.closing(subp.stdout) as stdout:
        if p.get("stdout_flag"):
            v = _parse_from_stdoutflag(stdout, p["stdout_flag"])
        elif p.get("paren_flag"):
            v = _parse_from_parenflag(stdout, p["paren_flag"])
        else:
            lines = [l.strip() for l in stdout.read().split("\n") if l.strip()]
            v = lines[-1]
    if v.endswith("."):
        v = v[:-1]
    return v

def _get_brew_versions():
    """Retrieve versions of tools installed via brew.
    """
    from bcbio import install
    tooldir = install.get_defaults().get("tooldir")
    brew_cmd = os.path.join(tooldir, "bin", "brew") if tooldir else "brew"
    try:
        vout = subprocess.check_output([brew_cmd, "which"])
        uses_which = True
    except subprocess.CalledProcessError:
        vout = subprocess.check_output([brew_cmd, "list", "--versions"])
        uses_which = False
    except OSError:  # brew not installed/used
        vout = ""
    out = {}
    for vstr in vout.split("\n"):
        if vstr.strip():
            if uses_which:
                name, v = vstr.rstrip().split(": ")
            else:
                parts = vstr.rstrip().split()
                name = parts[0]
                v = parts[-1]
            out[name] = v
    return out

def _get_versions(config=None):
    """Retrieve details on all programs available on the system.
    """
    out = [{"program": "bcbio-nextgen",
            "version": ("%s-%s" % (version.__version__, version.__git_revision__)
                        if version.__git_revision__ else version.__version__)}]
    manifest_vs = _get_versions_manifest()
    if manifest_vs:
        return out + manifest_vs
    else:
        assert config is not None, "Need configuration to retrieve from non-manifest installs"
        brew_vs = _get_brew_versions()
        import HTSeq
        out.append({"program": "htseq", "version": HTSeq.__version__})
        for p in _cl_progs:
            out.append({"program": p["cmd"],
                        "version": (brew_vs[p["cmd"]] if p["cmd"] in brew_vs else
                                    _get_cl_version(p, config))})
        for p in _alt_progs:
            out.append({"program": p["name"],
                        "version": (brew_vs[p["name"]] if p["name"] in brew_vs else
                                    p["version_fn"](config))})
        return out

def _get_versions_manifest():
    """Retrieve versions from a pre-existing manifest of installed software.
    """
    all_pkgs = ["htseq", "cn.mops", "vt", "platypus-variant", "gatk-framework"] + \
               [p.get("name", p["cmd"]) for p in _cl_progs] + [p["name"] for p in _alt_progs]
    manifest_dir = os.path.join(config_utils.get_base_installdir(), "manifest")
    if os.path.exists(manifest_dir):
        out = []
        for plist in ["toolplus", "brew", "python", "r", "debian", "custom"]:
            pkg_file = os.path.join(manifest_dir, "%s-packages.yaml" % plist)
            if os.path.exists(pkg_file):
                with open(pkg_file) as in_handle:
                    pkg_info = yaml.safe_load(in_handle)
                added = []
                for pkg in all_pkgs:
                    if pkg in pkg_info:
                        added.append(pkg)
                        out.append({"program": pkg, "version": pkg_info[pkg]["version"]})
                for x in added:
                    all_pkgs.remove(x)
        out.sort(key=lambda x: x["program"])
        for pkg in all_pkgs:
            out.append({"program": pkg, "version": ""})
        return out

def _get_program_file(dirs):
    if dirs.get("work"):
        base_dir = utils.safe_makedir(os.path.join(dirs["work"], "provenance"))
        return os.path.join(base_dir, "programs.txt")

def write_versions(dirs, config=None, is_wrapper=False):
    """Write CSV file with versions used in analysis pipeline.
    """
    out_file = _get_program_file(dirs)
    if is_wrapper:
        assert utils.file_exists(out_file), "Failed to create program versions from VM"
    elif out_file is None:
        for p in _get_versions(config):
            print("{program},{version}".format(**p))
    else:
        with open(out_file, "w") as out_handle:
            for p in _get_versions(config):
                out_handle.write("{program},{version}\n".format(**p))
    return out_file

def add_subparser(subparsers):
    """Add command line option for exporting version information.
    """
    parser = subparsers.add_parser("version",
                                   help="Export versions of used software to stdout or a file ")
    parser.add_argument("--workdir", help="Directory export programs to in workdir/provenance/programs.txt",
                        default=None)

def get_version(name, dirs=None, config=None):
    """Retrieve the current version of the given program from cached names.
    """
    if dirs:
        p = _get_program_file(dirs)
    else:
        p = config["resources"]["program_versions"]
    with open(p) as in_handle:
        for line in in_handle:
            prog, version = line.rstrip().split(",")
            if prog == name and version:
                return version
    raise KeyError("Version information not found for %s in %s" % (name, p))
<|endoftext|>


<commit_before>from decimal import Decimal
from mexbtcapi import log

def check_number_for_decimal_conversion(number):
    a= type(number) in (int, long)
    b= isinstance(number, (str, Decimal))
    if not (a or b):
        log.warning("You are using a number ("+str(number)+") that is not suitable to convert to Decimal!")

class Currency( object ):
    def __init__(self, name):
        self.name= name
    def __repr__(self):
        return self.name

class ExchangeRate(object):
    def __init__(self, c1, c2, exchange_rate):
        assert all([isinstance(x, Currency) for x in (c1,c2)])
        '''c2 = exchange_rate * c1. If associated with a market, c2 is
        the "buy" currency'''
        assert c1!=c2
        check_number_for_decimal_conversion( exchange_rate )
        self.c1, self.c2= c1, c2
        self.exchange_rate= Decimal( exchange_rate )

    def convert(self, amount):
        assert isinstance(amount, Amount)
        if self.c1==amount.currency:
            return Amount(amount.value*self.exchange_rate, self.c2)
        elif self.c2==amount.currency:
            return Amount((1/amount.value)*self.exchange_rate, self.c1)
        else:
            raise Exception("Can't exchange currencies with this ExchangeRate")

    def __cmp__(self, other):
        if not isinstance(other, ExchangeRate) or other.c1!=self.c1 or other.c2!=self.c2:
            raise ValueError("can't compare the two amounts", str(self), str(other))
        return cmp(self.exchange_rate,other.exchange_rate)

    def __repr__(self):
        return "%.3f %s/%s" % (self.exchange_rate, currency_dict[self.c1], currency_dict[self.c2])

class Amount(object):
    def __init__(self, value, currency):
        check_number_for_decimal_conversion( value )
        self.value, self.currency= Decimal(value), currency

    def convert(self, currencyequivalence, to_currency):
        if self.currency!=to_currency:
            currencyequivalence.convert(self)

    def clone():
        '''returns a copy of this amount'''
        return Amount( self.value, self.currency)

    def __repr__(self):
        return str(self.value)+" "+str(self.currency)

    def __iadd__(self, other):
        if type(other) in (int, float):
            self.value+=other
        elif isinstance(other, Amount):
            if self.currency!=other.currency:
                raise ValueError("Can't sum two amounts in different currencies")
            self.value+=other.value
        else:
            raise ValueError("Can't sum Amount to ", type(other))
        return self

    def __add__(self, other):
        a= self.clone()
        a+= other
        return a

    def __neg__(self):
        a= self.clone()
        a.value= -a.value
        return a

    def __isub__(self,other):
        self+= -other
        return self

    def __sub__(self, other):
        a= self.clone() + (-other)
        return a

    def __cmp__(self, other):
        if not isinstance(other, Amount) or other.currency!=self.currency:
            raise ValueError("can't compare the two amounts", str(self), str(other))
        return cmp(self.value,other.value)
<commit_msg>fix old ExchangeRate print code<commit_after>from decimal import Decimal
from mexbtcapi import log

def check_number_for_decimal_conversion(number):
    a= type(number) in (int, long)
    b= isinstance(number, (str, Decimal))
    if not (a or b):
        log.warning("You are using a number ("+str(number)+") that is not suitable to convert to Decimal!")

class Currency( object ):
    def __init__(self, name):
        self.name= name
    def __repr__(self):
        return self.name

class ExchangeRate(object):
    def __init__(self, c1, c2, exchange_rate):
        assert all([isinstance(x, Currency) for x in (c1,c2)])
        '''c2 = exchange_rate * c1. If associated with a market, c2 is
        the "buy" currency'''
        assert c1!=c2
        check_number_for_decimal_conversion( exchange_rate )
        self.c1, self.c2= c1, c2
        self.exchange_rate= Decimal( exchange_rate )

    def convert(self, amount):
        assert isinstance(amount, Amount)
        if self.c1==amount.currency:
            return Amount(amount.value*self.exchange_rate, self.c2)
        elif self.c2==amount.currency:
            return Amount((1/amount.value)*self.exchange_rate, self.c1)
        else:
            raise Exception("Can't exchange currencies with this ExchangeRate")

    def __cmp__(self, other):
        if not isinstance(other, ExchangeRate) or other.c1!=self.c1 or other.c2!=self.c2:
            raise ValueError("can't compare the two amounts", str(self), str(other))
        return cmp(self.exchange_rate,other.exchange_rate)

    def __repr__(self):
        return "%.3f %s/%s" % (self.exchange_rate, self.c2.name, self.c1.name)

class Amount(object):
    def __init__(self, value, currency):
        check_number_for_decimal_conversion( value )
        self.value, self.currency= Decimal(value), currency

    def convert(self, currencyequivalence, to_currency):
        if self.currency!=to_currency:
            currencyequivalence.convert(self)

    def clone():
        '''returns a copy of this amount'''
        return Amount( self.value, self.currency)

    def __repr__(self):
        return str(self.value)+" "+str(self.currency)

    def __iadd__(self, other):
        if type(other) in (int, float):
            self.value+=other
        elif isinstance(other, Amount):
            if self.currency!=other.currency:
                raise ValueError("Can't sum two amounts in different currencies")
            self.value+=other.value
        else:
            raise ValueError("Can't sum Amount to ", type(other))
        return self

    def __add__(self, other):
        a= self.clone()
        a+= other
        return a

    def __neg__(self):
        a= self.clone()
        a.value= -a.value
        return a

    def __isub__(self,other):
        self+= -other
        return self

    def __sub__(self, other):
        a= self.clone() + (-other)
        return a

    def __cmp__(self, other):
        if not isinstance(other, Amount) or other.currency!=self.currency:
            raise ValueError("can't compare the two amounts", str(self), str(other))
        return cmp(self.value,other.value)
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2016 Luc Saffre
# License: BSD (see file COPYING for details)

from lino.api import dd, rt, _
from lino.utils.xmlgen.html import E

class ChangeObservable(dd.Model):

    """
    Mixin for models which can "emit notifications" and define a list
    "observers" to be notified when an instance is modified.

    """

    class Meta(object):
        abstract = True

    def get_notify_subject(self, ar):
        return _("{user} modified {obj}").format(
            user=ar.get_user(), obj=self)

    def get_notify_body(self, ar):
        return E.tostring(E.p(
            _("{user} modified ").format(user=ar.get_user()),
            ar.obj2html(self),
            E.p("TODO: include a summary of the modifications.")))

    def get_notify_owner(self, ar):
        """E.g. comments.Comment overrides this. """
        return self

    def get_change_observers(self):
        """Override this in subclasses to yield a list of users who are
        observing changes on this object.

        """
        return []

    # def emit_notification(self, ar, owner, subject, body):
    #     return super(ChangeObservable, self).emit_notification(
    #         ar, owner, subject, body, self.get_notify_observers())

    def after_ui_save(self, ar, cw):

        super(ChangeObservable, self).after_ui_save(ar, cw)

        subject = self.get_notify_subject(ar)
        body = self.get_notify_body(ar)
        owner = self.get_notify_owner(ar)
        # self.emit_notification(ar, owner, subject, body)
        emit = rt.models.notify.Notification.emit_notification
        emit(ar, owner, subject, body, self.get_change_observers())

<commit_msg>API optimization for #1227<commit_after># -*- coding: UTF-8 -*-
# Copyright 2016 Luc Saffre
# License: BSD (see file COPYING for details)

from lino.api import dd, rt, _
from lino.utils.xmlgen.html import E

class ChangeObservable(dd.Model):

    """
    Mixin for models which can "emit notifications" and define a list
    "observers" to be notified when an instance is modified.

    """

    class Meta(object):
        abstract = True

    def get_notify_message(self, ar):
        return (self.get_notify_subject(ar), self.get_notify_body(ar))

    def get_notify_subject(self, ar):
        return _("{user} modified {obj}").format(
            user=ar.get_user(), obj=self)

    def get_notify_body(self, ar):
        return E.tostring(E.div(
            E.p(_("{user} modified ").format(user=ar.get_user()),
                ar.obj2html(self)),
            E.p("TODO: include a summary of the modifications.")))

    def get_notify_owner(self, ar):
        """E.g. comments.Comment overrides this. """
        return self

    def get_change_observers(self):
        """Override this in subclasses to yield a list of users who are
        observing changes on this object.

        """
        return []

    # def emit_notification(self, ar, owner, subject, body):
    #     return super(ChangeObservable, self).emit_notification(
    #         ar, owner, subject, body, self.get_notify_observers())

    def after_ui_save(self, ar, cw):

        super(ChangeObservable, self).after_ui_save(ar, cw)

        msg = self.get_notify_message(ar)
        if not msg:
            return
        subject, body = msg
        owner = self.get_notify_owner(ar)
        emit = rt.models.notify.Notification.emit_notification
        emit(ar, owner, subject, body, self.get_change_observers())

<|endoftext|>


<commit_before>import os
import json
import logging
import websocket
import requests
import time
import thread
import slacksocket.errors as errors
from .config import slackurl,event_types
from .models import SlackEvent,SlackMsg

log = logging.getLogger('slacksocket')

class SlackClient(requests.Session):
    """
    """
    def __init__(self,token):
        super(SlackClient, self).__init__()
        self.token = token

    def get_json(self,url):
        return self._result(self._get(url))

    def _post(self,url,payload=None):
        if payload:
            return self.post(url,params={'token':self.token},payload=payload)
        return self.post(url,params={'token':self.token})

    def _get(self,url):
        return self.get(url,params={'token':self.token})

    def _result(self,res):
        try:
            res.raise_for_status()
        except requests.exceptions.HTTPError as e:
            raise errors.SlackSocketAPIError(e)

        rj = res.json()
        if not rj['ok']:
            raise errors.SlackSocketAPIError('Error from slack api:\n %s' % r.text)

        return rj

class SlackSocket(object):
    """
    SlackSocket class provides a streaming interface to the Slack Real Time
    Messaging API
    params:
     - slacktoken(str): token to authenticate with slack
     - translate(bool): yield events with human-readable names
        rather than id. default true
    """
    def __init__(self,slacktoken,translate=True):
        if type(translate) != bool:
            raise TypeError('translate must be a boolean')
        self.eventq = []
        self.sendq = []
        self.translate = translate
        self.client = SlackClient(slacktoken)

        self.team,self.user = self._auth_test()
        self.thread = thread.start_new_thread(self._open,())

    def get_event(self,event_filter='all'):
        """
        return a single event object or block until an event is
        received and return it.
        params:
         - event_filter(list): Slack event type(s) to filter by. Excluding a
            filter returns all slack events. See https://api.slack.com/events
            for a listing of valid event types.
        """
        self._validate_filters(event_filter)

        #return or block until we have something to return
        while True:
            try:
                e = self.eventq.pop(0)
                #return immediately if no filtering
                if event_filter == 'all':
                    return e
                if e.type in event_filter:
                    return e
            except IndexError:
                time.sleep(.2)

    def events(self,event_filter='all'):
        """
        returns a blocking generator yielding Slack event objects
        params:
         - event_filter(list): Slack event type(s) to filter by. Excluding a
            filter returns all slack events. See https://api.slack.com/events
            for a listing of valid event types.
        """
        self._validate_filters(event_filter)

        while True:
            e = self.get_event(event_filter=event_filter)
            yield(e)

    def send_msg(self,text,channel_name=None,channel_id=None):
        """
        Send a message via Slack RTM socket, returning the message object
        after receiving a reply-to confirmation
        """
        if not channel_name and not channel_id:
            raise Exception('One of channel_id or channel_name \
                             parameters must be given')
        if channel_name:
            c = self._lookup_channel_by_name(channel_name)
            channel_id = c['channel_id']

        self.send_id += 1
        msg = SlackMsg(self.send_id,channel_id,text)
        self.ws.send(msg.json)
        for e in self.events():
            if e.event.has_key('reply_to'):
                if e.event['reply_to'] == self.send_id:
                    msg.sent = True
                    msg.ts = e.event['ts']
                    return msg
        return msg

    #######
    # Internal Methods
    #######

    def _open(self):
        #reset id for sending messages with each new socket
        self.send_id = 0
        self.ws = websocket.WebSocketApp(self._get_websocket_url(),
                                    on_message = self._event_handler,
                                    on_error   = self._error_handler,
                                    on_open    = self._open_handler,
                                    on_close   = self._exit_handler)
        self.ws.run_forever()

    def _validate_filters(self,filters):
        if filters == 'all':
            filters = event_types

        if type(filters) != list:
            raise TypeError('filters must be given as a list')

        for f in filters:
            if f not in event_types:
                raise SlackSocketEventNameError('unknown event type %s\n \
                             see https://api.slack.com/events' % filters)

    def _get_websocket_url(self):
        """
        Retrieve a fresh websocket url from slack api
        """
        return self.client.get_json(slackurl['rtm'])['url']

    def _auth_test(self):
        """
        Perform API auth test and get our user and team
        """
        test = self.client.get_json(slackurl['test'])

        if self.translate:
            return (test['team'],test['user'])
        else:
            return (test['team_id'],test['user_id'])

    def _lookup_user(self,user_id):
        """
        Look up a username from user id
        """
        if user_id == 'USLACKBOT':
            return "slackbot"

        members = self.client.get_json(slackurl['users'])['members']

        for user in members:
            if user['id'] == user_id:
                return user['name']
        else:
            return "unknown"

    def _lookup_channel_by_id(self,id):
        """
        Look up a channelname from channel id
        params:
         - id(str): The channel id to lookup
        """
        for ctype in ['channels','groups','ims']:
            channel_list = self.client.get_json(slackurl[ctype])[ctype]
            matching = [ c for c in channel_list if c['id'] == id ]
            if matching:
                channel = matching[0]
                if ctype == 'ims':
                    cname = self._lookup_user(channel['user'])
                else:
                    cname = channel['name']

                return { 'channel_type' : ctype,
                         'channel_name' : cname }

        #if no matches were found
        return { 'channel_type' : 'unknown',
                 'channel_name' : 'unknown' }

    def _lookup_channel_by_name(self,cname):
        """
        Look up a channel id from a given name
        params:
         - cname(str): The channel name to lookup
        """
        for ctype in ['channels','groups']:
            channel_list = self.client.get_json(slackurl[ctype])[ctype]
            matching = [ c for c in channel_list if c['name'] == cname ]
            if matching:
                channel = matching[0]

                return { 'channel_type' : ctype,
                         'channel_id'   : channel['id'] }

        #if no matches were found
        return { 'channel_type' : 'unknown',
                 'channel_id'   : 'unknown' }

    def _translate_event(self,event):
        """
        Translate all user and channel ids in an event to human-readable names
        """
        if event.event.has_key('user'):
            event.event['user'] = self._lookup_user(event.event['user'])

        if event.event.has_key('channel'):
            c = self._lookup_channel_by_id(event.event['channel'])
            event.event['channel'] = c['channel_name']

        event.mentions = [ self._lookup_user(u) for u in event.mentions ]

        return event

    #######
    # Websocket Handlers
    #######

    def _event_handler(self,ws,event_json):
        log.debug('event recieved: %s' % event_json)
        event = SlackEvent(event_json)

        #TODO: make use of ctype returned from _lookup_channel
        if self.translate:
            event = self._translate_event(event)

        self.eventq.append(event)

    def _open_handler(self,ws):
        log.info('websocket connection established')

    def _error_handler(self,ws,error):
        log.critical('websocket error:\n %s' % error)

    def _exit_handler(self,ws):
        self.thread = thread.start_new_thread(self.open,())
<commit_msg>add websocket reconnect logic<commit_after>import os
import json
import logging
import websocket
import requests
import time
import thread
import slacksocket.errors as errors
from .config import slackurl,event_types
from .models import SlackEvent,SlackMsg

log = logging.getLogger('slacksocket')

class SlackClient(requests.Session):
    """
    """
    def __init__(self,token):
        super(SlackClient, self).__init__()
        self.token = token

    def get_json(self,url):
        return self._result(self._get(url))

    def _post(self,url,payload=None):
        if payload:
            return self.post(url,params={'token':self.token},payload=payload)
        return self.post(url,params={'token':self.token})

    def _get(self,url):
        return self.get(url,params={'token':self.token})

    def _result(self,res):
        try:
            res.raise_for_status()
        except requests.exceptions.HTTPError as e:
            raise errors.SlackSocketAPIError(e)

        rj = res.json()
        if not rj['ok']:
            raise errors.SlackSocketAPIError('Error from slack api:\n %s' % r.text)

        return rj

class SlackSocket(object):
    """
    SlackSocket class provides a streaming interface to the Slack Real Time
    Messaging API
    params:
     - slacktoken(str): token to authenticate with slack
     - translate(bool): yield events with human-readable names
        rather than id. default true
    """
    def __init__(self,slacktoken,translate=True):
        if type(translate) != bool:
            raise TypeError('translate must be a boolean')
        self.eventq = []
        self.sendq = []
        self.translate = translate
        self.client = SlackClient(slacktoken)

        self.team,self.user = self._auth_test()
        self.thread = thread.start_new_thread(self._open,())

    def get_event(self,event_filter='all'):
        """
        return a single event object or block until an event is
        received and return it.
        params:
         - event_filter(list): Slack event type(s) to filter by. Excluding a
            filter returns all slack events. See https://api.slack.com/events
            for a listing of valid event types.
        """
        self._validate_filters(event_filter)

        #return or block until we have something to return
        while True:
            try:
                e = self.eventq.pop(0)
                #return immediately if no filtering
                if event_filter == 'all':
                    return e
                if e.type in event_filter:
                    return e
            except IndexError:
                time.sleep(.2)

    def events(self,event_filter='all'):
        """
        returns a blocking generator yielding Slack event objects
        params:
         - event_filter(list): Slack event type(s) to filter by. Excluding a
            filter returns all slack events. See https://api.slack.com/events
            for a listing of valid event types.
        """
        self._validate_filters(event_filter)

        while True:
            e = self.get_event(event_filter=event_filter)
            yield(e)

    def send_msg(self,text,channel_name=None,channel_id=None):
        """
        Send a message via Slack RTM socket, returning the message object
        after receiving a reply-to confirmation
        """
        if not channel_name and not channel_id:
            raise Exception('One of channel_id or channel_name \
                             parameters must be given')
        if channel_name:
            c = self._lookup_channel_by_name(channel_name)
            channel_id = c['channel_id']

        self.send_id += 1
        msg = SlackMsg(self.send_id,channel_id,text)
        self.ws.send(msg.json)
        for e in self.events():
            if e.event.has_key('reply_to'):
                if e.event['reply_to'] == self.send_id:
                    msg.sent = True
                    msg.ts = e.event['ts']
                    return msg
        return msg

    #######
    # Internal Methods
    #######

    def _open(self):
        #reset id for sending messages with each new socket
        self.send_id = 0
        self.ws = websocket.WebSocketApp(self._get_websocket_url(),
                                    on_message = self._event_handler,
                                    on_error   = self._error_handler,
                                    on_open    = self._open_handler,
                                    on_close   = self._exit_handler)
        self.ws.run_forever()

    def _validate_filters(self,filters):
        if filters == 'all':
            filters = event_types

        if type(filters) != list:
            raise TypeError('filters must be given as a list')

        for f in filters:
            if f not in event_types:
                raise SlackSocketEventNameError('unknown event type %s\n \
                             see https://api.slack.com/events' % filters)

    def _get_websocket_url(self):
        """
        Retrieve a fresh websocket url from slack api
        """
        return self.client.get_json(slackurl['rtm'])['url']

    def _auth_test(self):
        """
        Perform API auth test and get our user and team
        """
        test = self.client.get_json(slackurl['test'])

        if self.translate:
            return (test['team'],test['user'])
        else:
            return (test['team_id'],test['user_id'])

    def _lookup_user(self,user_id):
        """
        Look up a username from user id
        """
        if user_id == 'USLACKBOT':
            return "slackbot"

        members = self.client.get_json(slackurl['users'])['members']

        for user in members:
            if user['id'] == user_id:
                return user['name']
        else:
            return "unknown"

    def _lookup_channel_by_id(self,id):
        """
        Look up a channelname from channel id
        params:
         - id(str): The channel id to lookup
        """
        for ctype in ['channels','groups','ims']:
            channel_list = self.client.get_json(slackurl[ctype])[ctype]
            matching = [ c for c in channel_list if c['id'] == id ]
            if matching:
                channel = matching[0]
                if ctype == 'ims':
                    cname = self._lookup_user(channel['user'])
                else:
                    cname = channel['name']

                return { 'channel_type' : ctype,
                         'channel_name' : cname }

        #if no matches were found
        return { 'channel_type' : 'unknown',
                 'channel_name' : 'unknown' }

    def _lookup_channel_by_name(self,cname):
        """
        Look up a channel id from a given name
        params:
         - cname(str): The channel name to lookup
        """
        for ctype in ['channels','groups']:
            channel_list = self.client.get_json(slackurl[ctype])[ctype]
            matching = [ c for c in channel_list if c['name'] == cname ]
            if matching:
                channel = matching[0]

                return { 'channel_type' : ctype,
                         'channel_id'   : channel['id'] }

        #if no matches were found
        return { 'channel_type' : 'unknown',
                 'channel_id'   : 'unknown' }

    def _translate_event(self,event):
        """
        Translate all user and channel ids in an event to human-readable names
        """
        if event.event.has_key('user'):
            event.event['user'] = self._lookup_user(event.event['user'])

        if event.event.has_key('channel'):
            c = self._lookup_channel_by_id(event.event['channel'])
            event.event['channel'] = c['channel_name']

        event.mentions = [ self._lookup_user(u) for u in event.mentions ]

        return event

    #######
    # Websocket Handlers
    #######

    def _event_handler(self,ws,event_json):
        log.debug('event recieved: %s' % event_json)
        event = SlackEvent(event_json)

        #TODO: make use of ctype returned from _lookup_channel
        if self.translate:
            event = self._translate_event(event)

        self.eventq.append(event)

    def _open_handler(self,ws):
        log.info('websocket connection established')
        self.connect_ts = time.time()

    def _error_handler(self,ws,error):
        log.critical('websocket error:\n %s' % error)

    def _exit_handler(self,ws):
        log.warn('websocket connection closed')
        # Don't attempt reconnect if our last attempt was less than 10s ago
        if (time.time() - self.connect_ts) < 10:
            raise errors.SlackSocketConnectionError(
                    'Failed to establish a websocket connection'
                    )
        log.warn('attempting to reconnect')
        self.thread = thread.start_new_thread(self._open,())
<|endoftext|>


<commit_before>"""Prepare batch scripts for submitting bcbio_vm jobs.

Automates the process of preparing submission batch scripts, using the same arguments as
standard IPython runs.
"""
import os

from bcbiovm.docker import defaults

# names that indicate we're running on a dedicated AWS queue
AWS_QUEUES = set(["cloud"])

def _get_ipython_cmdline(args):
    """Translate arguments back into a standard bcbio_vm ipython submission command.
    """
    cmd = ["bcbio_vm.py", "ipython", args.sample_config, args.scheduler, args.queue,
           "--numcores", str(args.numcores)]
    has_timelimit = False
    for resource in args.resources:
        cmd += ["-r", resource]
        if resource.startswith("timelimit"):
            has_timelimit = True
    if not has_timelimit and args.queue in AWS_QUEUES:
        cmd += ["-r", "timelimit=0"]
    for opt_arg in ["timeout", "retries", "tag", "tmpdir", "fcdir", "systemconfig"]:
        if getattr(args, opt_arg):
            cmd += ["--%s" % opt_arg, str(getattr(args, opt_arg))]
    return " ".join(cmd)

def submit_script(args):
    args = defaults.update_check_args(args, "Could not prep batch scripts")
    out_file = os.path.join(os.getcwd(), "bcbio_submit.sh")
    with open(out_file, "w") as out_handle:
        out_handle.write("#!/bin/bash\n")
        out_handle.write(_get_scheduler_cmds(args) + "\n")
        out_handle.write(_get_ipython_cmdline(args) + "\n")
    print("Submission script for %s written to %s" % (args.scheduler, out_file))
    print("Start analysis with: %s %s" % (_get_submit_cmd(args.scheduler), out_file))

def _get_scheduler_cmds(args):
    cmds = {"slurm": _get_slurm_cmds,
            "sge": _get_sge_cmds,
            "lsf": _get_lsf_cmds,
            "torque": _get_torque_cmds,
            "pbspro": _get_torque_cmds}
    try:
        return cmds[args.scheduler](args)
    except KeyError:
        raise NotImplementedError("Batch script preparation for %s not yet supported" % args.scheduler)

def _get_slurm_cmds(args):
    cmds = ["--cpus-per-task=1", "--mem=2000", "-p %s" % args.queue]
    timelimit = "0" if args.queue in AWS_QUEUES else "1-00:00:00"
    for r in args.resources:
        if r.startswith("timelimit"):
            _, timelimit = r.split("=")
    cmds += ["-t %s" % timelimit]
    if args.tag:
        cmds += ["-J %s-submit" % args.tag]
    return "\n".join("#SBATCH %s" % x for x in cmds)

def _get_sge_cmds(args):
    cmds = ["-cwd", "-j y", "-S /bin/bash"]
    if args.queue:
        cmds += ["-q %s" % args.queue]
    if args.tag:
        cmds += ["-N %s-submit" % args.tag]
    return "\n".join("#$ %s" % x for x in cmds)

def _get_lsf_cmds(args):
    cmds = ["-q %s" % args.queue, "-n 1"]
    if args.tag:
        cmds += ["-J %s-submit" % args.tag]
    return "\n".join("#BSUB %s" % x for x in cmds)

def _get_torque_cmds(args):
    cmds = ["-V", "-j oe", "-q %s" % args.queue, "-l nodes=1:ppn=1"]
    if args.tag:
        cmds += ["-N %s-submit" % args.tag]
    return "\n".join("#PBS %s" % x for x in cmds)

def _get_submit_cmd(scheduler):
    cmds = {"slurm": "sbatch",
            "sge": "qsub",
            "lsf": "bsub",
            "torque": "qsub",
            "pbspro": "qsub"}
    return cmds[scheduler]
<commit_msg>Cosmetic changes for bcbiovm.ipython.batchprep<commit_after>"""Prepare batch scripts for submitting bcbio_vm jobs.

Automates the process of preparing submission batch scripts, using
the same arguments as standard IPython runs.
"""
import os

from bcbiovm.docker import defaults

# names that indicate we're running on a dedicated AWS queue
AWS_QUEUES = set(["cloud"])

def _get_ipython_cmdline(args):
    """Translate arguments back into a standard bcbio_vm ipython
    submission command.
    """
    cmd = ["bcbio_vm.py", "ipython", args.sample_config, args.scheduler,
           args.queue, "--numcores", str(args.numcores)]
    has_timelimit = False
    for resource in args.resources:
        cmd += ["-r", resource]
        if resource.startswith("timelimit"):
            has_timelimit = True
    if not has_timelimit and args.queue in AWS_QUEUES:
        cmd += ["-r", "timelimit=0"]
    for opt_arg in ["timeout", "retries", "tag", "tmpdir",
                    "fcdir", "systemconfig"]:
        if getattr(args, opt_arg):
            cmd += ["--%s" % opt_arg, str(getattr(args, opt_arg))]
    return " ".join(cmd)

def submit_script(args):
    args = defaults.update_check_args(args, "Could not prep batch scripts")
    out_file = os.path.join(os.getcwd(), "bcbio_submit.sh")
    with open(out_file, "w") as out_handle:
        out_handle.write("#!/bin/bash\n")
        out_handle.write(_get_scheduler_cmds(args) + "\n")
        out_handle.write(_get_ipython_cmdline(args) + "\n")
    print("Submission script for %s written to %s" %
          (args.scheduler, out_file))
    print("Start analysis with: %s %s" %
          (_get_submit_cmd(args.scheduler), out_file))

def _get_scheduler_cmds(args):
    cmds = {
        "slurm": _get_slurm_cmds,
        "sge": _get_sge_cmds,
        "lsf": _get_lsf_cmds,
        "torque": _get_torque_cmds,
        "pbspro": _get_torque_cmds
    }
    try:
        return cmds[args.scheduler](args)
    except KeyError:
        raise NotImplementedError("Batch script preparation for %s "
                                  "not yet supported" % args.scheduler)

def _get_slurm_cmds(args):
    cmds = ["--cpus-per-task=1", "--mem=2000", "-p %s" % args.queue]
    timelimit = "0" if args.queue in AWS_QUEUES else "1-00:00:00"
    for r in args.resources:
        if r.startswith("timelimit"):
            _, timelimit = r.split("=")
    cmds += ["-t %s" % timelimit]
    if args.tag:
        cmds += ["-J %s-submit" % args.tag]
    return "\n".join("#SBATCH %s" % x for x in cmds)

def _get_sge_cmds(args):
    cmds = ["-cwd", "-j y", "-S /bin/bash"]
    if args.queue:
        cmds += ["-q %s" % args.queue]
    if args.tag:
        cmds += ["-N %s-submit" % args.tag]
    return "\n".join("#$ %s" % x for x in cmds)

def _get_lsf_cmds(args):
    cmds = ["-q %s" % args.queue, "-n 1"]
    if args.tag:
        cmds += ["-J %s-submit" % args.tag]
    return "\n".join("#BSUB %s" % x for x in cmds)

def _get_torque_cmds(args):
    cmds = ["-V", "-j oe", "-q %s" % args.queue, "-l nodes=1:ppn=1"]
    if args.tag:
        cmds += ["-N %s-submit" % args.tag]
    return "\n".join("#PBS %s" % x for x in cmds)

def _get_submit_cmd(scheduler):
    cmds = {
        "slurm": "sbatch",
        "sge": "qsub",
        "lsf": "bsub",
        "torque": "qsub",
        "pbspro": "qsub"
    }
    return cmds[scheduler]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from csacompendium.utils.abstractmodels import (
    AuthUserDetail,
    CreateUpdateTime,
)
from csacompendium.utils.createslug import create_slug
from csacompendium.utils.modelmanagers import (
    model_instance_filter,
    model_type_filter,
    create_model_type,
)
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.signals import pre_save
from django.dispatch import receiver
from django.core.urlresolvers import reverse

class LocationManager(models.Manager):
    """
    Location model manager
    """
    def filter_by_instance(self, instance):
        """
        Query a related Location object/record from another model's object
        :param instance: Object instance
        :return: Query result from content type/model
        :rtye: object/record
        """
        return model_instance_filter(instance, self, LocationManager)

    def create_by_model_type(self, model_type, slug, **kwargs):
        """
        Create object by model type
        :param model_type: Content/model type
        :param slug: Slug
        :param kwargs: Fields to be created
        :return: Data object
        :rtype: Object
        """
        return create_model_type(self, model_type, slug, slugify=True, **kwargs)

class Location(AuthUserDetail, CreateUpdateTime):
    """
    Location model.  Creates location entity.
    """
    limit = models.Q(app_label='countries', model='country')
    slug = models.SlugField(unique=True, blank=True)
    content_type = models.ForeignKey(ContentType, on_delete=models.PROTECT, limit_choices_to=limit)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    location_name = models.CharField(max_length=256, blank=True)
    latitude = models.DecimalField(max_digits=8, decimal_places=6)
    longitude = models.DecimalField(max_digits=9, decimal_places=6)
    elevation = models.FloatField(blank=True, null=True)
    objects = LocationManager()

    def __unicode__(self):
        return self.location_name

    def __str__(self):
        return self.location_name

    def get_api_url(self):
        """
        Get location URL as a reverse from model
        :return: URL
        :rtype: String
        """
        return reverse('location_api:location_detail', kwargs={'slug': self.slug})

    class Meta:
        unique_together = ['latitude', 'longitude']
        ordering = ['-time_created', '-last_update']
        verbose_name_plural = 'Locations'

    @property
    def model_type_relation(self):
        """
        Get related LocationRelation object/record
        :return: Query result from the LocationRelation model
        :rtye: object/record
        """
        instance = self
        qs = LocationRelation.objects.filter_by_model_type(instance)
        return qs

@receiver(pre_save, sender=Location)
def pre_save_country_receiver(sender, instance, *args, **kwargs):
    """
    Create a slug before save.
    :param sender: Signal sending objec
    :param instance: Object instance
    :param args: Any other argument
    :param kwargs: Keyword arguments
    :return: None
    :rtype: None
    """
    if not instance.slug:
        instance.slug = create_slug(instance, Location, instance.location_name)

class LocationRelationManager(models.Manager):
    """
    Location relation model manager
    """
    def filter_by_instance(self, instance):
        """
        Query a related LocationRelation object/record from another model's object
        :param instance: Object instance
        :return: Query result from content type/model
        :rtye: object/record
        """
        return model_instance_filter(instance, self, LocationRelationManager)

    def filter_by_model_type(self, instance):
        """
        Query related objects/mode type
        :param instance: Object instance
        :return: Matching object else none
        :rtype: Object/record
        """
        obj_qs = super(LocationRelationManager, self).filter(location=instance.id)
        return model_type_filter(self, obj_qs, LocationRelationManager)

    def create_by_model_type(self, model_type, pk, **kwargs):
        """
        Create object by model type
        :param model_type: Content/model type
        :param pk: Primary key
        :param kwargs: Fields to be created
        :return: Data object
        :rtype: Object
        """
        return create_model_type(self, model_type, pk, slugify=False, **kwargs)

class LocationRelation(AuthUserDetail, CreateUpdateTime):
    """
    Location entry relationship model. A many to many bridge table between location  model
    and other models
    """
    limit = models.Q(app_label='locations', model='temperature') | models.Q(app_label='locations', model='location')
    location = models.ForeignKey(Location, on_delete=models.PROTECT)
    content_type = models.ForeignKey(ContentType, on_delete=models.PROTECT, limit_choices_to=limit)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    objects = LocationRelationManager()

    class Meta:
        ordering = ['-time_created', '-last_update']

class Temperature(AuthUserDetail, CreateUpdateTime):
    """
    Temperature model. Creates temperature entity
    """
    temperature = models.DecimalField(max_digits=5, decimal_places=2)
    temperature_uom = models.CharField(max_length=5, default='°C')

    def __unicode__(self):
        return str(self.temperature)

    def __str__(self):
        return str(self.temperature)

    def get_api_url(self):
        """
        Get temperature URL as a reverse from model
        :return: URL
        :rtype: String
        """
        return reverse('location_api:temperature_detail', kwargs={'pk': self.pk})

    class Meta:
        ordering = ['-time_created', '-last_update']
        verbose_name_plural = 'Temperatures'

    @property
    def location_relations(self):
        """
        Get related LocationRelation object/record
        :return: Query result from the LocationRelation model
        :rtye: object/record
        """
        instance = self
        qs = LocationRelation.objects.filter_by_instance(instance)
        return qs

<commit_msg>Added a verbose name<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from csacompendium.utils.abstractmodels import (
    AuthUserDetail,
    CreateUpdateTime,
)
from csacompendium.utils.createslug import create_slug
from csacompendium.utils.modelmanagers import (
    model_instance_filter,
    model_type_filter,
    create_model_type,
)
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.signals import pre_save
from django.dispatch import receiver
from django.core.urlresolvers import reverse

class LocationManager(models.Manager):
    """
    Location model manager
    """
    def filter_by_instance(self, instance):
        """
        Query a related Location object/record from another model's object
        :param instance: Object instance
        :return: Query result from content type/model
        :rtye: object/record
        """
        return model_instance_filter(instance, self, LocationManager)

    def create_by_model_type(self, model_type, slug, **kwargs):
        """
        Create object by model type
        :param model_type: Content/model type
        :param slug: Slug
        :param kwargs: Fields to be created
        :return: Data object
        :rtype: Object
        """
        return create_model_type(self, model_type, slug, slugify=True, **kwargs)

class Location(AuthUserDetail, CreateUpdateTime):
    """
    Location model.  Creates location entity.
    """
    limit = models.Q(app_label='countries', model='country')
    slug = models.SlugField(unique=True, blank=True)
    content_type = models.ForeignKey(ContentType, on_delete=models.PROTECT, limit_choices_to=limit)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    location_name = models.CharField(max_length=256, blank=True)
    latitude = models.DecimalField(max_digits=8, decimal_places=6)
    longitude = models.DecimalField(max_digits=9, decimal_places=6)
    elevation = models.FloatField(blank=True, null=True)
    objects = LocationManager()

    def __unicode__(self):
        return self.location_name

    def __str__(self):
        return self.location_name

    def get_api_url(self):
        """
        Get location URL as a reverse from model
        :return: URL
        :rtype: String
        """
        return reverse('location_api:location_detail', kwargs={'slug': self.slug})

    class Meta:
        unique_together = ['latitude', 'longitude']
        ordering = ['-time_created', '-last_update']
        verbose_name_plural = 'Locations'

    @property
    def model_type_relation(self):
        """
        Get related LocationRelation object/record
        :return: Query result from the LocationRelation model
        :rtye: object/record
        """
        instance = self
        qs = LocationRelation.objects.filter_by_model_type(instance)
        return qs

@receiver(pre_save, sender=Location)
def pre_save_country_receiver(sender, instance, *args, **kwargs):
    """
    Create a slug before save.
    :param sender: Signal sending objec
    :param instance: Object instance
    :param args: Any other argument
    :param kwargs: Keyword arguments
    :return: None
    :rtype: None
    """
    if not instance.slug:
        instance.slug = create_slug(instance, Location, instance.location_name)

class LocationRelationManager(models.Manager):
    """
    Location relation model manager
    """
    def filter_by_instance(self, instance):
        """
        Query a related LocationRelation object/record from another model's object
        :param instance: Object instance
        :return: Query result from content type/model
        :rtye: object/record
        """
        return model_instance_filter(instance, self, LocationRelationManager)

    def filter_by_model_type(self, instance):
        """
        Query related objects/mode type
        :param instance: Object instance
        :return: Matching object else none
        :rtype: Object/record
        """
        obj_qs = super(LocationRelationManager, self).filter(location=instance.id)
        return model_type_filter(self, obj_qs, LocationRelationManager)

    def create_by_model_type(self, model_type, pk, **kwargs):
        """
        Create object by model type
        :param model_type: Content/model type
        :param pk: Primary key
        :param kwargs: Fields to be created
        :return: Data object
        :rtype: Object
        """
        return create_model_type(self, model_type, pk, slugify=False, **kwargs)

class LocationRelation(AuthUserDetail, CreateUpdateTime):
    """
    Location entry relationship model. A many to many bridge table between location  model
    and other models
    """
    limit = models.Q(app_label='locations', model='temperature') | models.Q(app_label='locations', model='location')
    location = models.ForeignKey(Location, on_delete=models.PROTECT)
    content_type = models.ForeignKey(ContentType, on_delete=models.PROTECT, limit_choices_to=limit)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    objects = LocationRelationManager()

    class Meta:
        ordering = ['-time_created', '-last_update']

class Temperature(AuthUserDetail, CreateUpdateTime):
    """
    Temperature model. Creates temperature entity
    """
    temperature = models.DecimalField(max_digits=5, decimal_places=2)
    temperature_uom = models.CharField(max_length=5, default='°C', verbose_name='Temperature UOM')

    def __unicode__(self):
        return str(self.temperature)

    def __str__(self):
        return str(self.temperature)

    def get_api_url(self):
        """
        Get temperature URL as a reverse from model
        :return: URL
        :rtype: String
        """
        return reverse('location_api:temperature_detail', kwargs={'pk': self.pk})

    class Meta:
        ordering = ['-time_created', '-last_update']
        verbose_name_plural = 'Temperatures'

    @property
    def location_relations(self):
        """
        Get related LocationRelation object/record
        :return: Query result from the LocationRelation model
        :rtye: object/record
        """
        instance = self
        qs = LocationRelation.objects.filter_by_instance(instance)
        return qs

<|endoftext|>


<commit_before># Copyright 2016 Sean Robertson

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Pytests for `pydrobert.kaldi` logging """

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import os

from io import StringIO
from tempfile import NamedTemporaryFile

import numpy as np
import pytest

import pydrobert.kaldi as kaldi

@pytest.fixture
def temp_file_1_name():
    temp = NamedTemporaryFile(delete=False)
    temp.close()
    yield temp.name
    os.remove(temp.name)

@pytest.fixture
def logger():
    ret_logger = logging.getLogger('pydrobert.kaldi')
    s_stream = StringIO()
    handler = logging.StreamHandler(s_stream)
    ret_logger.addHandler(handler)
    old_makeRecord = ret_logger.makeRecord
    yield ret_logger
    ret_logger.makeRecord = old_makeRecord
    ret_logger.removeHandler(handler)
    ret_logger.setLevel(logging.NOTSET)

def test_basic_write(logger):
    s_stream = logger.handlers[-1].stream
    assert not s_stream.tell()
    test_string = "I'm a dude playing a dude disguised as another dude"
    logger.warning(test_string)
    assert s_stream.tell()
    assert test_string + '\n' == s_stream.getvalue()

def elicit_warning(filename, threaded=False):
    # helper to elicit a warning from kaldi
    writer = kaldi.tables.open('ark,t:{}'.format(filename), 'bv', 'w')
    writer.write('zz', [np.infty])
    writer.close()
    reader = kaldi.tables.open(
        'ark,t{}:{}'.format(',bg' if threaded else '', filename), 'bv')
    next(reader)
    reader.close()

@pytest.mark.parametrize('threaded', [True, False])
def test_elicit_kaldi_warning(logger, temp_file_1_name, threaded):
    s_stream = logger.handlers[-1].stream
    assert not s_stream.tell()
    elicit_warning(temp_file_1_name, threaded)
    assert s_stream.tell()
    assert 'Reading infinite value into vector.\n' == s_stream.getvalue()

def test_set_level(logger, temp_file_1_name):
    s_stream = logger.handlers[-1].stream
    assert not s_stream.tell()
    rwspecifier = 'ark,t:{}'.format(temp_file_1_name)
    logger.setLevel(logging.ERROR)
    logger.warning('sound of silence')
    assert not s_stream.tell()
    elicit_warning(temp_file_1_name)
    assert not s_stream.tell()
    logger.setLevel(logging.INFO)
    elicit_warning(temp_file_1_name)
    assert s_stream.tell()
    assert 'Reading infinite value into vector.\n' == s_stream.getvalue()

def test_log_source_is_appropriate(logger, temp_file_1_name):
    handler = logger.handlers[-1]
    s_stream = handler.stream
    rwspecifier = 'ark,t:{}'.format(temp_file_1_name)
    handler.setFormatter(logging.Formatter('%(filename)s: %(message)s'))
    assert not s_stream.tell()
    logger.warning('pokeymans')
    assert 'test_logging.py' in s_stream.getvalue()
    assert 'kaldi-vector.cc' not in s_stream.getvalue()
    s_stream.seek(0)
    elicit_warning(temp_file_1_name)
    assert 'kaldi-vector.cc' in s_stream.getvalue()
    assert '__init__.py' not in s_stream.getvalue()

def test_python_error_doesnt_segfault(logger, temp_file_1_name):
    def _raise_exception(*args, **kwargs):
        raise Exception()
    logger.makeRecord = _raise_exception
    with pytest.raises(Exception):
        logger.warning('foo')
    with pytest.raises(Exception):
        elicit_warning(temp_file_1_name)
<commit_msg>Wasn't a problem with the package; was a problem with the tests<commit_after># Copyright 2016 Sean Robertson

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Pytests for `pydrobert.kaldi` logging """

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import os

try:
    from StringIO import StringIO # py 2.7
except ImportError:
    from io import StringIO
from tempfile import NamedTemporaryFile

import numpy as np
import pytest

import pydrobert.kaldi as kaldi

@pytest.fixture
def temp_file_1_name():
    temp = NamedTemporaryFile(delete=False)
    temp.close()
    yield temp.name
    os.remove(temp.name)

@pytest.fixture
def logger():
    ret_logger = logging.getLogger('pydrobert.kaldi')
    s_stream = StringIO()
    handler = logging.StreamHandler(s_stream)
    ret_logger.addHandler(handler)
    old_makeRecord = ret_logger.makeRecord
    yield ret_logger
    ret_logger.makeRecord = old_makeRecord
    ret_logger.removeHandler(handler)
    ret_logger.setLevel(logging.NOTSET)

def test_basic_write(logger):
    s_stream = logger.handlers[-1].stream
    assert not s_stream.tell()
    test_string = "I'm a dude playing a dude disguised as another dude"
    logger.warning(test_string)
    assert s_stream.tell()
    assert test_string + '\n' == s_stream.getvalue()

def elicit_warning(filename, threaded=False):
    # helper to elicit a warning from kaldi
    writer = kaldi.tables.open('ark,t:{}'.format(filename), 'bv', 'w')
    writer.write('zz', [np.infty])
    writer.close()
    reader = kaldi.tables.open(
        'ark,t{}:{}'.format(',bg' if threaded else '', filename), 'bv')
    next(reader)
    reader.close()

@pytest.mark.parametrize('threaded', [True, False])
def test_elicit_kaldi_warning(logger, temp_file_1_name, threaded):
    s_stream = logger.handlers[-1].stream
    assert not s_stream.tell()
    elicit_warning(temp_file_1_name, threaded)
    assert s_stream.tell()
    assert 'Reading infinite value into vector.\n' == s_stream.getvalue()

def test_set_level(logger, temp_file_1_name):
    s_stream = logger.handlers[-1].stream
    assert not s_stream.tell()
    rwspecifier = 'ark,t:{}'.format(temp_file_1_name)
    logger.setLevel(logging.ERROR)
    logger.warning('sound of silence')
    assert not s_stream.tell()
    elicit_warning(temp_file_1_name)
    assert not s_stream.tell()
    logger.setLevel(logging.INFO)
    elicit_warning(temp_file_1_name)
    assert s_stream.tell()
    assert 'Reading infinite value into vector.\n' == s_stream.getvalue()

def test_log_source_is_appropriate(logger, temp_file_1_name):
    handler = logger.handlers[-1]
    s_stream = handler.stream
    rwspecifier = 'ark,t:{}'.format(temp_file_1_name)
    handler.setFormatter(logging.Formatter('%(filename)s: %(message)s'))
    assert not s_stream.tell()
    logger.warning('pokeymans')
    assert 'test_logging.py' in s_stream.getvalue()
    assert 'kaldi-vector.cc' not in s_stream.getvalue()
    s_stream.seek(0)
    elicit_warning(temp_file_1_name)
    assert 'kaldi-vector.cc' in s_stream.getvalue()
    assert '__init__.py' not in s_stream.getvalue()

def test_python_error_doesnt_segfault(logger, temp_file_1_name):
    def _raise_exception(*args, **kwargs):
        raise Exception()
    logger.makeRecord = _raise_exception
    with pytest.raises(Exception):
        logger.warning('foo')
    with pytest.raises(Exception):
        elicit_warning(temp_file_1_name)
<|endoftext|>


<commit_before>
import sys

from django.conf import settings

from mezzanine.settings.models import Setting

registry = {}

def register_setting(name="", editable=False, description="", default=None):
    """
    Registers a setting that can be edited via the admin.
    """
    default = getattr(settings, "MEZZANINE_%s" % name, default)
    registry[name] = {"name": name, "description": description, "editable":
        editable, "default": default, "type": type(default)}

def editable_settings():
    """
    Returns the names of editable settings.
    """
    return [k for (k, v) in registry.items() if v["editable"]]

def load_settings(*names):
    """
    Returns a settings object using the given settings names as attributes.
    Values are loaded when the first name is accessed. Settings are loaded
    from the DB for settings marked as editable, and from
    ``mezzanine.settings.registry`` for those not retrieved from the DB.
    """

    class Settings(object):

        def __init__(self):
            self._loaded = False

        def __getattr__(self, name):
            if self._loaded:
                raise AttributeError("Setting does not exist: %s" % name)
            self._loaded = True
            editable = editable_settings()
            syncdb = len(sys.argv) >= 2 and sys.argv[1] == "syncdb"
            if editable and not syncdb:
                for setting in Setting.objects.filter(name__in=editable):
                    setting_type = registry[setting.name]["type"]
                    if setting_type is bool:
                        setting_value = setting.value != "False"
                    else:
                        setting_value = setting_type(setting.value)
                    setattr(self, setting.name, setting_value)
            for n in names:
                if n not in self.__dict__ and n in registry.keys():
                    setattr(self, n, registry[n]["default"])
            return getattr(self, name)

    return Settings()

for app in settings.INSTALLED_APPS:
    try:
        __import__("%s.defaults" % app)
    except ImportError:
        pass
<commit_msg>Fix unrequested settings being loaded from DB.<commit_after>
import sys

from django.conf import settings

from mezzanine.settings.models import Setting

registry = {}

def register_setting(name="", editable=False, description="", default=None):
    """
    Registers a setting that can be edited via the admin.
    """
    default = getattr(settings, "MEZZANINE_%s" % name, default)
    registry[name] = {"name": name, "description": description, "editable":
        editable, "default": default, "type": type(default)}

def editable_settings(names):
    """
    Returns the names of editable settings.
    """
    return [k for (k, v) in registry.items() if v["editable"] and k in names]

def load_settings(*names):
    """
    Returns a settings object using the given settings names as attributes.
    Values are loaded when the first name is accessed. Settings are loaded
    from the DB for settings marked as editable, and from
    ``mezzanine.settings.registry`` for those not retrieved from the DB.
    """

    class Settings(object):

        def __init__(self):
            self._loaded = False

        def __getattr__(self, name):
            if self._loaded:
                raise AttributeError("Setting does not exist: %s" % name)
            self._loaded = True
            editable = editable_settings(names)
            syncdb = len(sys.argv) >= 2 and sys.argv[1] == "syncdb"
            if editable and not syncdb:
                for setting in Setting.objects.filter(name__in=editable):
                    setting_type = registry[setting.name]["type"]
                    if setting_type is bool:
                        setting_value = setting.value != "False"
                    else:
                        setting_value = setting_type(setting.value)
                    setattr(self, setting.name, setting_value)
            for n in names:
                if n not in self.__dict__ and n in registry.keys():
                    setattr(self, n, registry[n]["default"])
            return getattr(self, name)

    return Settings()

for app in settings.INSTALLED_APPS:
    try:
        __import__("%s.defaults" % app)
    except ImportError:
        pass
<|endoftext|>