<commit_before># -*- coding: utf-8-*-
"""
A Speaker handles audio output from Jasper to the user

Speaker methods:
    say - output 'phrase' as speech
    play - play the audio in 'filename'
    is_available - returns True if the platform supports this implementation
"""
import os
import platform
import re
import sys
import tempfile
import subprocess
import pipes
import logging
from abc import ABCMeta, abstractmethod
from distutils.spawn import find_executable

import yaml
import argparse

import wave
try:
    import mad
    import gtts
except ImportError:
    pass

class AbstractTTSEngine(object):
    """
    Generic parent class for all speakers
    """
    __metaclass__ = ABCMeta

    @classmethod
    @abstractmethod
    def is_available(cls):
        return (find_executable('aplay') is not None)

    def __init__(self, **kwargs):
        self._logger = logging.getLogger(__name__)

    @abstractmethod
    def say(self, phrase, *args):
        pass

    def play(self, filename):
        # FIXME: Use platform-independent audio-output here
        # See issue jasperproject/jasper-client#188
        cmd = ['aplay', str(filename)]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)

class AbstractMp3TTSEngine(AbstractTTSEngine):
    """
    Generic class that implements the 'play' method for mp3 files
    """
    @classmethod
    def is_available(cls):
        return (super(AbstractMp3TTSEngine, cls).is_available() and 'mad' in sys.modules.keys())

    def play_mp3(self, filename):
        mf = mad.MadFile(filename)
        with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f:
            wav = wave.open(f, mode='wb')
            wav.setframerate(mf.samplerate())
            wav.setnchannels(1 if mf.mode() == mad.MODE_SINGLE_CHANNEL else 2)
            wav.setsampwidth(4L) # width of 32 bit audio
            frame = mf.read()
            while frame is not None:
                wav.writeframes(frame)
                frame = mf.read()
            wav.close()
            self.play(f.name)

class DummyTTS(AbstractTTSEngine):
    """
    Dummy TTS engine that logs phrases with INFO level instead of synthesizing
    speech.
    """

    SLUG = "dummy-tts"

    @classmethod
    def is_available(cls):
        return True

    def say(self, phrase):
        self._logger.info(phrase)

    def play(self, filename):
        self._logger.debug("Playback of file '%s' requested")
        pass

class EspeakTTS(AbstractTTSEngine):
    """
    Uses the eSpeak speech synthesizer included in the Jasper disk image
    Requires espeak to be available
    """

    SLUG = "espeak-tts"

    def __init__(self, voice='default+m3', pitch_adjustment=40, words_per_minute=160):
        super(self.__class__, self).__init__()
        self.voice = voice
        self.pitch_adjustment = pitch_adjustment
        self.words_per_minute = words_per_minute

    @classmethod
    def is_available(cls):
        return (super(cls, cls).is_available() and find_executable('espeak') is not None)

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f:
            fname = f.name
        cmd = ['espeak', '-v', self.voice,
                         '-p', self.pitch_adjustment,
                         '-s', self.words_per_minute,
                         '-w', fname,
                         phrase]
        cmd = [str(x) for x in cmd]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)
        self.play(fname)
        os.remove(fname)

class FestivalTTS(AbstractTTSEngine):
    """
    Uses the festival speech synthesizer
    Requires festival (text2wave) to be available
    """

    SLUG = 'festival-tts'

    @classmethod
    def is_available(cls):
        if super(cls, cls).is_available() and find_executable('text2wave') is not None and find_executable('festival') is not None:
            logger = logging.getLogger(__name__)
            cmd = ['festival', '--pipe']
            with tempfile.SpooledTemporaryFile() as out_f:
                with tempfile.SpooledTemporaryFile() as in_f:
                    logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
                    subprocess.call(cmd, stdin=in_f, stdout=out_f, stderr=out_f)
                    out_f.seek(0)
                    output = out_f.read().strip()
                    if output:
                        logger.debug("Output was: '%s'", output)
                    return ('No default voice found' not in output)
        return False

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        cmd = ['text2wave']
        with tempfile.NamedTemporaryFile(suffix='.wav') as out_f:
            with tempfile.SpooledTemporaryFile() as in_f:
                in_f.write(phrase)
                in_f.seek(0)
                with tempfile.SpooledTemporaryFile() as err_f:
                    self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
                    subprocess.call(cmd, stdin=in_f, stdout=out_f, stderr=err_f)
                    err_f.seek(0)
                    output = err_f.read()
                    if output:
                        self._logger.debug("Output was: '%s'", output)
            self.play(out_f.name)

class MacOSXTTS(AbstractTTSEngine):
    """
    Uses the OS X built-in 'say' command
    """

    SLUG = "osx-tts"

    @classmethod
    def is_available(cls):
        return (platform.system() == 'darwin' and find_executable('say') is not None and find_executable('afplay') is not None)

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        cmd = ['say', str(phrase)]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)

    def play(self, filename):
        cmd = ['afplay', str(filename)]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)

class PicoTTS(AbstractTTSEngine):
    """
    Uses the svox-pico-tts speech synthesizer
    Requires pico2wave to be available
    """

    SLUG = "pico-tts"

    def __init__(self, language="en-US"):
        super(self.__class__, self).__init__()
        self.language = language

    @classmethod
    def is_available(cls):
        return (super(cls, cls).is_available() and find_executable('pico2wave') is not None)

    @property
    def languages(self):
        cmd = ['pico2wave', '-l', 'NULL',
                            '-w', os.devnull,
                            'NULL']
        with tempfile.SpooledTemporaryFile() as f:
            subprocess.call(cmd, stderr=f)
            f.seek(0)
            output = f.read()
        pattern = re.compile(r'Unknown language: NULL\nValid languages:\n((?:[a-z]{2}-[A-Z]{2}\n)+)')
        matchobj = pattern.match(output)
        if not matchobj:
            raise RuntimeError("pico2wave: valid languages not detected")
        langs = matchobj.group(1).split()
        return langs

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f:
            fname = f.name
        cmd = ['pico2wave', '--wave', fname]
        if self.language not in self.languages:
                raise ValueError("Language '%s' not supported by '%s'", self.language, self.SLUG)
        cmd.extend(['-l', self.language])
        cmd.append(phrase)
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)
        self.play(fname)
        os.remove(fname)

class GoogleTTS(AbstractMp3TTSEngine):
    """
    Uses the Google TTS online translator
    Requires pymad and gTTS to be available
    """

    SLUG = "google-tts"

    def __init__(self, language='en'):
        super(self.__class__, self).__init__()
        self.language = language

    @classmethod
    def is_available(cls):
        return (super(cls, cls).is_available() and 'gtts' in sys.modules.keys())

    @property
    def languages(self):
        langs = ['af', 'sq', 'ar', 'hy', 'ca', 'zh-CN', 'zh-TW', 'hr', 'cs', 'da', 'nl', 'en', 'eo', 'fi', 'fr', 'de',
                 'el', 'ht', 'hi', 'hu', 'is', 'id', 'it', 'ja', 'ko', 'la', 'lv', 'mk', 'no', 'pl', 'pt', 'ro', 'ru',
                 'sr', 'sk', 'es', 'sw', 'sv', 'ta', 'th', 'tr', 'vi', 'cy']
        return langs

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        if self.language not in self.languages:
            raise ValueError("Language '%s' not supported by '%s'", self.language, self.SLUG)
        tts = gtts.gTTS(text=phrase, lang=self.language)
        with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f:
            tmpfile = f.name
        tts.save(tmpfile)
        self.play_mp3(tmpfile)
        os.remove(tmpfile)

def get_default_engine_slug():
    return 'osx-tts' if platform.system() == 'darwin' else 'espeak-tts'

def get_engine_by_slug(slug=None):
    """
    Returns:
        A speaker implementation available on the current platform

    Raises:
        ValueError if no speaker implementation is supported on this platform
    """

    if not slug or type(slug) is not str:
        raise TypeError("Invalid slug '%s'", slug)

    selected_engines = filter(lambda engine: hasattr(engine, "SLUG") and engine.SLUG == slug, get_engines())
    if len(selected_engines) == 0:
        raise ValueError("No TTS engine found for slug '%s'" % slug)
    else:
        if len(selected_engines) > 1:
            print("WARNING: Multiple TTS engines found for slug '%s'. This is most certainly a bug." % slug)
        engine = selected_engines[0]
        if not engine.is_available():
            raise ValueError("TTS engine '%s' is not available (due to missing dependencies, missing dependencies, etc.)" % slug)
        return engine

def get_engines():
    def get_subclasses(cls):
        subclasses = set()
        for subclass in cls.__subclasses__():
            subclasses.add(subclass)
            subclasses.update(get_subclasses(subclass))
        return subclasses
    return [tts_engine for tts_engine in list(get_subclasses(AbstractTTSEngine)) if hasattr(tts_engine, 'SLUG') and tts_engine.SLUG]

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Jasper TTS module')
    parser.add_argument('--debug', action='store_true', help='Show debug messages')
    args = parser.parse_args()

    logging.basicConfig()
    if args.debug:
        logger = logging.getLogger(__name__)
        logger.setLevel(logging.DEBUG)

    engines = get_engines()
    available_engines = []
    for engine in get_engines():
        if engine.is_available():
            available_engines.append(engine)
    print("Available TTS engines:")
    for i, engine in enumerate(available_engines, start=1):
        print("%d. %s" % (i, engine.SLUG))

    print("")
    print("Disabled TTS engines:")
    for i, engine in enumerate(list(set(engines).difference(set(available_engines))), start=1):
        print("%d. %s" % (i, engine.SLUG))

    print("")
    for i, engine in enumerate(available_engines, start=1):
        print("%d. Testing engine '%s'..." % (i, engine.SLUG))
        engine().say("This is a test.")
    print("Done.")
<commit_msg>Delete tempfiles in play_mp3() method of AbstractMp3TTSEngine<commit_after># -*- coding: utf-8-*-
"""
A Speaker handles audio output from Jasper to the user

Speaker methods:
    say - output 'phrase' as speech
    play - play the audio in 'filename'
    is_available - returns True if the platform supports this implementation
"""
import os
import platform
import re
import sys
import tempfile
import subprocess
import pipes
import logging
from abc import ABCMeta, abstractmethod
from distutils.spawn import find_executable

import yaml
import argparse

import wave
try:
    import mad
    import gtts
except ImportError:
    pass

class AbstractTTSEngine(object):
    """
    Generic parent class for all speakers
    """
    __metaclass__ = ABCMeta

    @classmethod
    @abstractmethod
    def is_available(cls):
        return (find_executable('aplay') is not None)

    def __init__(self, **kwargs):
        self._logger = logging.getLogger(__name__)

    @abstractmethod
    def say(self, phrase, *args):
        pass

    def play(self, filename):
        # FIXME: Use platform-independent audio-output here
        # See issue jasperproject/jasper-client#188
        cmd = ['aplay', str(filename)]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)

class AbstractMp3TTSEngine(AbstractTTSEngine):
    """
    Generic class that implements the 'play' method for mp3 files
    """
    @classmethod
    def is_available(cls):
        return (super(AbstractMp3TTSEngine, cls).is_available() and 'mad' in sys.modules.keys())

    def play_mp3(self, filename):
        mf = mad.MadFile(filename)
        with tempfile.NamedTemporaryFile(suffix='.wav') as f:
            wav = wave.open(f, mode='wb')
            wav.setframerate(mf.samplerate())
            wav.setnchannels(1 if mf.mode() == mad.MODE_SINGLE_CHANNEL else 2)
            wav.setsampwidth(4L) # width of 32 bit audio
            frame = mf.read()
            while frame is not None:
                wav.writeframes(frame)
                frame = mf.read()
            wav.close()
            self.play(f.name)

class DummyTTS(AbstractTTSEngine):
    """
    Dummy TTS engine that logs phrases with INFO level instead of synthesizing
    speech.
    """

    SLUG = "dummy-tts"

    @classmethod
    def is_available(cls):
        return True

    def say(self, phrase):
        self._logger.info(phrase)

    def play(self, filename):
        self._logger.debug("Playback of file '%s' requested")
        pass

class EspeakTTS(AbstractTTSEngine):
    """
    Uses the eSpeak speech synthesizer included in the Jasper disk image
    Requires espeak to be available
    """

    SLUG = "espeak-tts"

    def __init__(self, voice='default+m3', pitch_adjustment=40, words_per_minute=160):
        super(self.__class__, self).__init__()
        self.voice = voice
        self.pitch_adjustment = pitch_adjustment
        self.words_per_minute = words_per_minute

    @classmethod
    def is_available(cls):
        return (super(cls, cls).is_available() and find_executable('espeak') is not None)

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f:
            fname = f.name
        cmd = ['espeak', '-v', self.voice,
                         '-p', self.pitch_adjustment,
                         '-s', self.words_per_minute,
                         '-w', fname,
                         phrase]
        cmd = [str(x) for x in cmd]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)
        self.play(fname)
        os.remove(fname)

class FestivalTTS(AbstractTTSEngine):
    """
    Uses the festival speech synthesizer
    Requires festival (text2wave) to be available
    """

    SLUG = 'festival-tts'

    @classmethod
    def is_available(cls):
        if super(cls, cls).is_available() and find_executable('text2wave') is not None and find_executable('festival') is not None:
            logger = logging.getLogger(__name__)
            cmd = ['festival', '--pipe']
            with tempfile.SpooledTemporaryFile() as out_f:
                with tempfile.SpooledTemporaryFile() as in_f:
                    logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
                    subprocess.call(cmd, stdin=in_f, stdout=out_f, stderr=out_f)
                    out_f.seek(0)
                    output = out_f.read().strip()
                    if output:
                        logger.debug("Output was: '%s'", output)
                    return ('No default voice found' not in output)
        return False

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        cmd = ['text2wave']
        with tempfile.NamedTemporaryFile(suffix='.wav') as out_f:
            with tempfile.SpooledTemporaryFile() as in_f:
                in_f.write(phrase)
                in_f.seek(0)
                with tempfile.SpooledTemporaryFile() as err_f:
                    self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
                    subprocess.call(cmd, stdin=in_f, stdout=out_f, stderr=err_f)
                    err_f.seek(0)
                    output = err_f.read()
                    if output:
                        self._logger.debug("Output was: '%s'", output)
            self.play(out_f.name)

class MacOSXTTS(AbstractTTSEngine):
    """
    Uses the OS X built-in 'say' command
    """

    SLUG = "osx-tts"

    @classmethod
    def is_available(cls):
        return (platform.system() == 'darwin' and find_executable('say') is not None and find_executable('afplay') is not None)

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        cmd = ['say', str(phrase)]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)

    def play(self, filename):
        cmd = ['afplay', str(filename)]
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)

class PicoTTS(AbstractTTSEngine):
    """
    Uses the svox-pico-tts speech synthesizer
    Requires pico2wave to be available
    """

    SLUG = "pico-tts"

    def __init__(self, language="en-US"):
        super(self.__class__, self).__init__()
        self.language = language

    @classmethod
    def is_available(cls):
        return (super(cls, cls).is_available() and find_executable('pico2wave') is not None)

    @property
    def languages(self):
        cmd = ['pico2wave', '-l', 'NULL',
                            '-w', os.devnull,
                            'NULL']
        with tempfile.SpooledTemporaryFile() as f:
            subprocess.call(cmd, stderr=f)
            f.seek(0)
            output = f.read()
        pattern = re.compile(r'Unknown language: NULL\nValid languages:\n((?:[a-z]{2}-[A-Z]{2}\n)+)')
        matchobj = pattern.match(output)
        if not matchobj:
            raise RuntimeError("pico2wave: valid languages not detected")
        langs = matchobj.group(1).split()
        return langs

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f:
            fname = f.name
        cmd = ['pico2wave', '--wave', fname]
        if self.language not in self.languages:
                raise ValueError("Language '%s' not supported by '%s'", self.language, self.SLUG)
        cmd.extend(['-l', self.language])
        cmd.append(phrase)
        self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd]))
        with tempfile.TemporaryFile() as f:
            subprocess.call(cmd, stdout=f, stderr=f)
            f.seek(0)
            output = f.read()
            if output:
                self._logger.debug("Output was: '%s'", output)
        self.play(fname)
        os.remove(fname)

class GoogleTTS(AbstractMp3TTSEngine):
    """
    Uses the Google TTS online translator
    Requires pymad and gTTS to be available
    """

    SLUG = "google-tts"

    def __init__(self, language='en'):
        super(self.__class__, self).__init__()
        self.language = language

    @classmethod
    def is_available(cls):
        return (super(cls, cls).is_available() and 'gtts' in sys.modules.keys())

    @property
    def languages(self):
        langs = ['af', 'sq', 'ar', 'hy', 'ca', 'zh-CN', 'zh-TW', 'hr', 'cs', 'da', 'nl', 'en', 'eo', 'fi', 'fr', 'de',
                 'el', 'ht', 'hi', 'hu', 'is', 'id', 'it', 'ja', 'ko', 'la', 'lv', 'mk', 'no', 'pl', 'pt', 'ro', 'ru',
                 'sr', 'sk', 'es', 'sw', 'sv', 'ta', 'th', 'tr', 'vi', 'cy']
        return langs

    def say(self, phrase):
        self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
        if self.language not in self.languages:
            raise ValueError("Language '%s' not supported by '%s'", self.language, self.SLUG)
        tts = gtts.gTTS(text=phrase, lang=self.language)
        with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f:
            tmpfile = f.name
        tts.save(tmpfile)
        self.play_mp3(tmpfile)
        os.remove(tmpfile)

def get_default_engine_slug():
    return 'osx-tts' if platform.system() == 'darwin' else 'espeak-tts'

def get_engine_by_slug(slug=None):
    """
    Returns:
        A speaker implementation available on the current platform

    Raises:
        ValueError if no speaker implementation is supported on this platform
    """

    if not slug or type(slug) is not str:
        raise TypeError("Invalid slug '%s'", slug)

    selected_engines = filter(lambda engine: hasattr(engine, "SLUG") and engine.SLUG == slug, get_engines())
    if len(selected_engines) == 0:
        raise ValueError("No TTS engine found for slug '%s'" % slug)
    else:
        if len(selected_engines) > 1:
            print("WARNING: Multiple TTS engines found for slug '%s'. This is most certainly a bug." % slug)
        engine = selected_engines[0]
        if not engine.is_available():
            raise ValueError("TTS engine '%s' is not available (due to missing dependencies, missing dependencies, etc.)" % slug)
        return engine

def get_engines():
    def get_subclasses(cls):
        subclasses = set()
        for subclass in cls.__subclasses__():
            subclasses.add(subclass)
            subclasses.update(get_subclasses(subclass))
        return subclasses
    return [tts_engine for tts_engine in list(get_subclasses(AbstractTTSEngine)) if hasattr(tts_engine, 'SLUG') and tts_engine.SLUG]

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Jasper TTS module')
    parser.add_argument('--debug', action='store_true', help='Show debug messages')
    args = parser.parse_args()

    logging.basicConfig()
    if args.debug:
        logger = logging.getLogger(__name__)
        logger.setLevel(logging.DEBUG)

    engines = get_engines()
    available_engines = []
    for engine in get_engines():
        if engine.is_available():
            available_engines.append(engine)
    print("Available TTS engines:")
    for i, engine in enumerate(available_engines, start=1):
        print("%d. %s" % (i, engine.SLUG))

    print("")
    print("Disabled TTS engines:")
    for i, engine in enumerate(list(set(engines).difference(set(available_engines))), start=1):
        print("%d. %s" % (i, engine.SLUG))

    print("")
    for i, engine in enumerate(available_engines, start=1):
        print("%d. Testing engine '%s'..." % (i, engine.SLUG))
        engine().say("This is a test.")
    print("Done.")
<|endoftext|>


<commit_before>from igf_data.utils.dbutils import read_dbconf_json
from igf_data.task_tracking.igf_slack import IGF_slack
from igf_data.task_tracking.igf_asana import IGF_asana

class Reset_samplesheet_md5:
  '''
  A class for modifying samplesheet md5 for seqrun data processing
  '''
  def __init__(self,seqrun_path,seqrun_igf_list,dbconfig_file,
               json_collection_type='ILLUMINA_BCL_MD5',log_slack=True,
               log_asana=True,slack_config=None,asana_project_id=None,
               asana_config=None):
    '''
    :param seqrun_path: A directory path for sequencing run home
    :param seqrun_igf_list: A file path listing sequencing runs to reset
    :param dbconfig_file: A file containing the database configuration
    :param json_collection_type: A collection type for md5 json file lookup, default ILLUMINA_BCL_MD5
    :param log_slack: A boolean flag for toggling Slack messages, default True
    :param log_asana: Aboolean flag for toggling Asana message, default True
    :param slack_config: A file containing Slack tokens, default None
    :param asana_config: A file containing Asana tokens, default None
    :param asana_project_id: A numeric Asana project id, default is None
    '''
    try:
      self.seqrun_path=seqrun_path
      self.seqrun_igf_list=seqrun_igf_list
      self.json_collection_type=json_collection_type
      self.log_slack=log_slack
      self.log_asana=log_asana
      dbparams = read_dbconf_json(dbconfig_file)
      base=BaseAdaptor(**dbparams)
      self.session_class = base.get_session_class()                             # add session class to instance
      if log_slack and slack_config is None:
        raise ValueError('Missing slack config file')
      elif log_slack and slack_config:
        self.igf_slack = IGF_slack(slack_config)                                # add slack object

      if log_asana and \
         (asana_config is None or \
          asana_project_id is None):
        raise ValueError('Missing asana config file or asana project id')
      elif log_asana and asana_config and asana_project_id:
        self.igf_asana=IGF_asana(asana_config,asana_project_id)                 # add asana object
    except:
      raise<commit_msg>reading list of seqrun ids from a file to a list<commit_after>import os
from igf_data.utils.dbutils import read_dbconf_json
from igf_data.task_tracking.igf_slack import IGF_slack
from igf_data.task_tracking.igf_asana import IGF_asana

class Reset_samplesheet_md5:
  '''
  A class for modifying samplesheet md5 for seqrun data processing
  '''
  def __init__(self,seqrun_path,seqrun_igf_list,dbconfig_file,
               json_collection_type='ILLUMINA_BCL_MD5',log_slack=True,
               log_asana=True,slack_config=None,asana_project_id=None,
               asana_config=None):
    '''
    :param seqrun_path: A directory path for sequencing run home
    :param seqrun_igf_list: A file path listing sequencing runs to reset
    :param dbconfig_file: A file containing the database configuration
    :param json_collection_type: A collection type for md5 json file lookup, default ILLUMINA_BCL_MD5
    :param log_slack: A boolean flag for toggling Slack messages, default True
    :param log_asana: Aboolean flag for toggling Asana message, default True
    :param slack_config: A file containing Slack tokens, default None
    :param asana_config: A file containing Asana tokens, default None
    :param asana_project_id: A numeric Asana project id, default is None
    '''
    try:
      self.seqrun_path=seqrun_path
      self.seqrun_igf_list=self._read_seqrun_list(seqrun_igf_list)
      self.json_collection_type=json_collection_type
      self.log_slack=log_slack
      self.log_asana=log_asana
      dbparams = read_dbconf_json(dbconfig_file)
      base=BaseAdaptor(**dbparams)
      self.session_class = base.get_session_class()                             # add session class to instance
      if log_slack and slack_config is None:
        raise ValueError('Missing slack config file')
      elif log_slack and slack_config:
        self.igf_slack = IGF_slack(slack_config)                                # add slack object

      if log_asana and \
         (asana_config is None or \
          asana_project_id is None):
        raise ValueError('Missing asana config file or asana project id')
      elif log_asana and asana_config and asana_project_id:
        self.igf_asana=IGF_asana(asana_config,asana_project_id)                 # add asana object
    except:
      raise

  @staticmethod
  def _read_seqrun_list(seqrun_igf_list):
    '''
    A static method for reading list of sequencing run ids from a n input file
    to a list
    :param seqrun_igf_list: A file containing the sequencing run ids
    :return list: A list of seqrun ids from the input file
    '''
    try:
      if not os.path.exists(seqrun_igf_list):
        raise IOError('File {0} not found'.format(seqrun_igf_list))

      seqrun_ids=list()                                                         # define an empty list of seqrun ids
      with open(seqrun_igf_list,'r') as fp:
        seqrun_ids=[i.strip() for i in fp]                                      # add seqrun ids to the list
      return seqrun_ids
    except:
      raise<|endoftext|>


<commit_before>from pynrfjprog import API, Hex
import unittest
import sys
import os

"""
Get the full path to the hex file under test based on the operating system used.
"""
from sys import platform as _platform

if _platform == "linux" or _platform == "linux2" or _platform == "darwin": # Linux or Mac assumes use of armgcc.
    relativePathBasedOnOs = 'armgcc/nvmc.hex'
elif _platform == "win32": # And windows assumes use of Keil.
    relativePathBasedOnOs = 'arm\\Objects\\example.hex'

HEXFILE = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), relativePathBasedOnOs)
assert (os.path.isfile(HEXFILE)), "Error: must compile the example project before running test.py. Make sure name of hex file and path are correct in test.py"

class NVMCTests(unittest.TestCase):

    @classmethod
    def setUpClass(self):
        """
        Set up the test suite and initialize connection with target device.
        Define class variables here that encapsulate expected behavior of program under test.
        """
        self.expectedTestData     = 0x0 # These are the same variables defined in main.c of our nvmc example.
        self.uicrCustomerAddress0 = 0x10001080
        self.codeFlashAddress     = 0x1FFF8
        self.codeFlashPageAddress = 0x20000

        self.device_family = API.DeviceFamily.NRF51  # Start out with nRF51 as expected target device, will be checked and changed if target is an nRF52 series device.
        self.api = API.API(self.device_family)

        self._connect_to_target()

        self.device_version = self.api.read_device_version() # Check our assumption that the target was an nRF51 series device.
        if 'NRF52' in self.device_version: # If target is an nRF52 series device reconnect w/ correct device parameters.
            self.device_family = API.DeviceFamily.NRF52
            self.api.close()
            self._connect_to_target()

        self.api.recover() # Erase all Flash, RAM and disable any protection before running test suite.

    @classmethod
    def tearDownClass(self):
        self.api.recover()
        self.api.disconnect_from_emu()
        self.api.close()

    def setUp(self):
        pass

    def tearDown(self):
        self.api.erase_all()

    def test_erase_uicr(self):
        uicrCustomerAddress1 = self.uicrCustomerAddress0 + 4
        testData             = 0xDEADBEEF
        expectedData         = 0xFFFFFFFF

        self.api.write_u32(uicrCustomerAddress1, testData, 1)

        self._program_hex_file_and_run(HEXFILE)

        readData = self.api.read_u32(uicrCustomerAddress1)
        self.assertEqual(expectedData, readData)

    def test_write_code_flash(self):
        expectedData = [0x00] * 8
        self._program_hex_file_and_run(HEXFILE)

        readData = self.api.read(self.codeFlashAddress, 8) # Read 8 bytes (two registers) starting at codeFlashAddress.
        self.assertEqual(expectedData, readData)

    def test_code_flash_page_erased(self):
        expectedData = [0xFF] * 8

        self._program_hex_file_and_run(HEXFILE)

        readData = self.api.read(self.codeFlashPageAddress, 8) # Read 8 bytes (two registers) starting at codePageAddress.
        self.assertEqual(expectedData, readData)

    def test_uicr_written(self):
        expectedData = 0x0

        self._program_hex_file_and_run(HEXFILE)

        readData = self.api.read_u32(self.uicrCustomerAddress0)
        self.assertEqual(expectedData, readData)

    @classmethod
    def _connect_to_target(self):
        """
        Initialzes and connects self.api to target. self.api can now perform operations on connected device.
        """
        self.api.open()
        self.api.connect_to_emu_without_snr()
        self.api.connect_to_device()

    @classmethod
    def _program_hex_file_and_run(self, hex_file_path):
        """
        Programs hex_file to the target device, reset, and run the application.
        """
        data = Hex.Hex(hex_file_path)
        for segment in data:
            self.api.write(segment.address, segment.data, True)

        self.api.sys_reset()
        self.api.go()

unittest.main(verbosity = 2) # Call the unit tests with desired verbosity.
<commit_msg>not going to have test.py for all examples for now. instead wil make a specific example showing how to use pynrfjprog in test fixutre<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import

from django.conf import settings

import os
import logging
from celery import states
from celery.utils.log import get_task_logger
from dbaas.celery import app

from util import call_script
from system.models import Configuration
from notification.models import TaskHistory

LOG = get_task_logger(__name__)

CLONE_DATABASE_SCRIPT_NAME="dummy_clone.sh"

def get_history_for_task_id(task_id):
    try:
        return TaskHistory.objects.get(task_id=task_id)
    except Exception, e:
        LOG.error("could not find history for task id %s" % task_id)
        return None

@app.task(bind=True)
def clone_database(self, origin_database, dest_database, user=None):

    #register History
    task_history = TaskHistory.register(request=self.request, user=user)

    LOG.info("origin_database: %s" % origin_database)
    LOG.info("dest_database: %s" % dest_database)

    LOG.info("id: %s | task: %s | kwargs: %s | args: %s" % (self.request.id,
                                                            self.request.task,
                                                            self.request.kwargs,
                                                            str(self.request.args)))

    #origin
    origin_instance=origin_database.databaseinfra.instances.all()[0]

    db_orig=origin_database.name
    user_orig=origin_database.databaseinfra.user
    pass_orig=origin_database.databaseinfra.password
    host_orig=origin_instance.address
    port_orig=origin_instance.port

    #destination
    dest_instance=dest_database.databaseinfra.instances.all()[0]

    db_dest=dest_database.name
    user_dest=dest_database.databaseinfra.user
    pass_dest=dest_database.databaseinfra.password
    host_dest=dest_instance.address
    port_dest=dest_instance.port

    path_of_dump=Configuration.get_by_name('database_clone_dir')
    engine=origin_database.databaseinfra.engine.name

    args=[db_orig, user_orig, pass_orig, host_orig, str(int(port_orig)),
            db_dest, user_dest, pass_dest, host_dest, str(int(port_dest)),
            path_of_dump, engine
    ]
    call_script(CLONE_DATABASE_SCRIPT_NAME, working_dir=settings.SCRIPTS_PATH, args=args)

    task_history.update_status_for(TaskHistory.STATUS_FINISHED)
    return<commit_msg>try except task<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import

from django.conf import settings

import os
import logging
from celery import states
from celery.utils.log import get_task_logger
from dbaas.celery import app

from util import call_script
from system.models import Configuration
from notification.models import TaskHistory

LOG = get_task_logger(__name__)

CLONE_DATABASE_SCRIPT_NAME="dummy_clone.sh"

def get_history_for_task_id(task_id):
    try:
        return TaskHistory.objects.get(task_id=task_id)
    except Exception, e:
        LOG.error("could not find history for task id %s" % task_id)
        return None

@app.task(bind=True)
def clone_database(self, origin_database, dest_database, user=None):

    #register History
    task_history = TaskHistory.register(request=self.request, user=user)

    LOG.info("origin_database: %s" % origin_database)
    LOG.info("dest_database: %s" % dest_database)

    LOG.info("id: %s | task: %s | kwargs: %s | args: %s" % (self.request.id,
                                                            self.request.task,
                                                            self.request.kwargs,
                                                            str(self.request.args)))

    #origin
    origin_instance=origin_database.databaseinfra.instances.all()[0]

    db_orig=origin_database.name
    user_orig=origin_database.databaseinfra.user
    pass_orig=origin_database.databaseinfra.password
    host_orig=origin_instance.address
    port_orig=origin_instance.port

    #destination
    dest_instance=dest_database.databaseinfra.instances.all()[0]

    db_dest=dest_database.name
    user_dest=dest_database.databaseinfra.user
    pass_dest=dest_database.databaseinfra.password
    host_dest=dest_instance.address
    port_dest=dest_instance.port

    path_of_dump=Configuration.get_by_name('database_clone_dir')
    engine=origin_database.databaseinfra.engine.name

    args=[db_orig, user_orig, pass_orig, host_orig, str(int(port_orig)),
            db_dest, user_dest, pass_dest, host_dest, str(int(port_dest)),
            path_of_dump, engine
    ]

    try:
        call_script(CLONE_DATABASE_SCRIPT_NAME, working_dir=settings.SCRIPTS_PATH, args=args)
        task_history.update_status_for(TaskHistory.STATUS_FINISHED)
    except Exception, e:
        LOG.error("task id %s error: %s" % (self.request.id, e))
        task_history.update_status_for(TaskHistory.STATUS_PENDING)

    return<|endoftext|>


<commit_before>#!/usr/bin/env python
##
##  Copyright 2015 SRI International
##  License: https://ncanda.sri.com/software-license.txt
##
##  $Revision: 2110 $
##  $LastChangedBy: nicholsn $
##  $LastChangedDate: 2015-08-07 09:10:29 -0700 (Fri, 07 Aug 2015) $
##
"""
NP Release Script
======================
Generates a CSV file for all subjects included in NP analysis
"""
import os

import pandas as pd

directory = "/fs/ncanda-share/releases/NCANDA_DATA_00019/summaries"

nps_file = ["ataxia.csv", "cddr.csv", "clinical.csv", "cnp.csv", "dd100.csv",
            "dd1000.csv", "grooved_pegboard.csv", "ishihara.csv",
            "landoltc.csv", "rey-o.csv", "wais4.csv", "wrat4.csv",
            "mr_session_report.csv"]

final_df = pd.read_csv(os.path.join(directory, "demographics.csv"),
                       index_col=['subject','arm','visit'])

race_map = dict(native_american_american_indian=1,
                asian=2,
                pacific_islander=3,
                african_american_black=4,
                caucasian_white=5);

# Create a series for each race where there is a 1 or 0 indicating if the
# participant belongs to the race or not and append to the final_df
for k, v in race_map.iteritems():
    race_filter = final_df.race == v
    final_df[k] = race_filter.apply(lambda x: 1 if x == True else 0)

for i in nps_file:
    df = pd.read_csv(os.path.join(directory, i),
                     index_col=['subject','arm','visit'])
    final_df = pd.concat([final_df, df], axis=1)

final_df = final_df.rename(columns={'cddr31':'cddr_past_month_binge',
                                    'cddr30':'cddr_past_year_binge'})

def replace_binge_groups_1(x):
	"""
	 A binary variable from CDDR_PastYr_Binge (i.e., 0 or blank,
	 but not missing = 0, and 1 or more = 1)
	"""
	if x > 0:
		result = 1
	elif x == 0:
		result = 0
	else:
		result = pd.np.NaN
	return result

def replace_binge_groups_month(x):
	"""
	 #3groups based on past month binge drinking (0 - is o or blank, but not missing)
	"""
	if x == 0:
		result = 'None'
	elif x == 1:
		result = '1'
	elif x > 1:
		result = '2+'
	else:
		result = ''
	return result

final_df['binge_groups_1'] = final_df['cddr_past_year_binge'].apply(replace_binge_groups_1)
final_df['binge_groups_month'] = final_df['cddr_past_month_binge'].apply(replace_binge_groups_month)

final_df['exceed_or_mrianomaly'] = pd.np.NaN
final_df['hi_ed'] = pd.np.NaN

final_df = final_df[['site', 'sex', 'visit_age',
'exceed_or_mrianomaly', 'exceeds_bl_drinking', 'cddr_past_month_binge',
'cddr_past_year_binge', 'binge_groups_1', 'binge_groups_month', 'hispanic', 'race',
'race_label', 'mri_analysisanomalies', 'hi_ed', 'african_american_black', 'asian',
'caucasian_white', 'native_american_american_indian', 'pacific_islander',
'cddrheight_inches', 'cddrweight_pounds', 'ses_parent_yoe', 'fh_alc_density',
'fh_drug_density', 'bmi_value', 'bmi_zscore', 'bmi_percentile', 'pds_score',
'np_atax_sht_sum', 'np_atax_steps_sum', 'np_atax_standr_sum',
'np_atax_standl_sum', 'cnp_mpract_mp2rtcr', 'cnp_cpf_ifac_tot',
'cnp_cpf_ifac_rtc', 'cnp_cpw_iwrd_tot', 'cnp_cpw_iwrd_rtc',
'cnp_spcptnl_scpl_tprt', 'cnp_spcptnl_scpt_tp', 'cnp_spcptnl_scpt_tprt',
'cnp_sfnb2_sfnb_rtc', 'cnp_sfnb2_sfnb_rtc0', 'cnp_sfnb2_sfnb_rtc1',
'cnp_sfnb2_sfnb_rtc2', 'cnp_sfnb2_sfnb_mcr', 'cnp_sfnb2_sfnb_mrtc',
'cnp_pmat24a_pmat24_a_cr', 'cnp_pmat24a_pmat24_a_rtcr', 'cnp_cpfd_cpfdtp',
'cnp_cpfd_cpfdtprt', 'cnp_cpwd_dwrd_tot', 'cnp_cpwd_dwrd_rtc',
'cnp_shortvolt_svt', 'cnp_shortvolt_svtcrt', 'cnp_shortvolt_svt_eff',
'cnp_er40d_er40_cr', 'cnp_er40d_er40_crt', 'cnp_er40d_er40angrt',
'cnp_er40d_er40fearrt', 'cnp_er40d_er40haprt', 'cnp_er40d_er40noert',
'cnp_er40d_er40sadrt', 'cnp_pcet_pcetcr', 'cnp_pcet_pcetrtcr',
'cnp_pcet_pcet_eff', 'cnp_pcet_pcet_acc', 'cnp_pcet_pcet_acc2',
'cnp_medf36_medf36_hap_rtcr', 'cnp_medf36_medf36_sad_rtcr',
'cnp_medf36_medf36_ang_rtcr', 'cnp_medf36_medf36_fear_rtcr',
'cnp_medf36_medf36_a', 'cnp_medf36_medf36_t', 'cnp_pvoc_pvoccr',
'cnp_pvoc_pvocrtcr', 'cnp_pvrt_pvrtcr', 'cnp_pvrt_pvrtrtcr',
'cnp_pvrt_pvrt_eff', 'cnp_svdelay_svt_ld', 'cnp_svdelay_svtldrtc',
'cnp_svdelay_svtldrter', 'cnp_svdelay_svt_effd', 'cnp_sfnb2_sfnb_mrt',
'cnp_sfnb2_sfnb_meff', 'np_gpeg_dh_time', 'np_gpeg_ndh_time',
'np_wais4_rawscore', 'np_wrat4_wr_raw', 'np_wrat4_mc_raw',
'cnp_spcptnl_scpn_tp', 'cnp_spcptnl_scpn_tprt', 'cnp_spcptnl_scpl_tp',
'cnp_cpfd_dfac_tot', 'cnp_cpfd_dfac_rtc']]

final_df.to_csv('np_release.csv')
<commit_msg>UPDATE: NP Reporting Script to include Parent Report<commit_after>#!/usr/bin/env python
##
##  Copyright 2015 SRI International
##  License: https://ncanda.sri.com/software-license.txt
##
##  $Revision: 2110 $
##  $LastChangedBy: nicholsn $
##  $LastChangedDate: 2015-08-07 09:10:29 -0700 (Fri, 07 Aug 2015) $
##
"""
NP Release Script
======================
Generates a CSV file for all subjects included in NP analysis
"""
import os

import pandas as pd

directory = "/fs/ncanda-share/releases/NCANDA_DATA_00019/summaries"

nps_file = ["ataxia.csv", "cddr.csv", "clinical.csv", "cnp.csv", "dd100.csv",
            "dd1000.csv", "grooved_pegboard.csv", "ishihara.csv",
            "landoltc.csv", "rey-o.csv", "wais4.csv", "wrat4.csv",
            "mr_session_report.csv", "parentreport.csv"]

final_df = pd.read_csv(os.path.join(directory, "demographics.csv"),
                       index_col=['subject','arm','visit'])

race_map = dict(native_american_american_indian=1,
                asian=2,
                pacific_islander=3,
                african_american_black=4,
                caucasian_white=5);

# Create a series for each race where there is a 1 or 0 indicating if the
# participant belongs to the race or not and append to the final_df
for k, v in race_map.iteritems():
    race_filter = final_df.race == v
    final_df[k] = race_filter.apply(lambda x: 1 if x == True else 0)

for i in nps_file:
    df = pd.read_csv(os.path.join(directory, i),
                     index_col=['subject','arm','visit'])
    final_df = pd.concat([final_df, df], axis=1)

final_df = final_df.rename(columns={'cddr31':'cddr_past_month_binge',
                                    'cddr30':'cddr_past_year_binge'})

def replace_binge_groups_1(x):
	"""
	 A binary variable from CDDR_PastYr_Binge (i.e., 0 or blank,
	 but not missing = 0, and 1 or more = 1)
	"""
	if x > 0:
		result = 1
	elif x == 0:
		result = 0
	else:
		result = pd.np.NaN
	return result

def replace_binge_groups_month(x):
	"""
	 #3groups based on past month binge drinking (0 - is o or blank, but not missing)
	"""
	if x == 0:
		result = 'None'
	elif x == 1:
		result = '1'
	elif x > 1:
		result = '2+'
	else:
		result = ''
	return result

final_df['binge_groups_1'] = final_df['cddr_past_year_binge'].apply(replace_binge_groups_1)
final_df['binge_groups_month'] = final_df['cddr_past_month_binge'].apply(replace_binge_groups_month)

final_df['exceed_or_mrianomaly'] = pd.np.NaN
final_df['hi_ed'] = pd.np.NaN

final_df = final_df[['site', 'sex', 'visit_age',
'exceed_or_mrianomaly', 'exceeds_bl_drinking', 'cddr_past_month_binge',
'cddr_past_year_binge', 'binge_groups_1', 'binge_groups_month', 'hispanic', 'race',
'race_label', 'mri_analysisanomalies', 'hi_ed', 'african_american_black', 'asian',
'caucasian_white', 'native_american_american_indian', 'pacific_islander',
'cddrheight_inches', 'cddrweight_pounds', 'ses_parent_yoe', 'fh_alc_density',
'fh_drug_density', 'bmi_value', 'bmi_zscore', 'bmi_percentile', 'pds_score',
'np_atax_sht_sum', 'np_atax_steps_sum', 'np_atax_standr_sum',
'np_atax_standl_sum', 'cnp_mpract_mp2rtcr', 'cnp_cpf_ifac_tot',
'cnp_cpf_ifac_rtc', 'cnp_cpw_iwrd_tot', 'cnp_cpw_iwrd_rtc',
'cnp_spcptnl_scpl_tprt', 'cnp_spcptnl_scpt_tp', 'cnp_spcptnl_scpt_tprt',
'cnp_sfnb2_sfnb_rtc', 'cnp_sfnb2_sfnb_rtc0', 'cnp_sfnb2_sfnb_rtc1',
'cnp_sfnb2_sfnb_rtc2', 'cnp_sfnb2_sfnb_mcr', 'cnp_sfnb2_sfnb_mrtc',
'cnp_pmat24a_pmat24_a_cr', 'cnp_pmat24a_pmat24_a_rtcr', 'cnp_cpfd_cpfdtp',
'cnp_cpfd_cpfdtprt', 'cnp_cpwd_dwrd_tot', 'cnp_cpwd_dwrd_rtc',
'cnp_shortvolt_svt', 'cnp_shortvolt_svtcrt', 'cnp_shortvolt_svt_eff',
'cnp_er40d_er40_cr', 'cnp_er40d_er40_crt', 'cnp_er40d_er40angrt',
'cnp_er40d_er40fearrt', 'cnp_er40d_er40haprt', 'cnp_er40d_er40noert',
'cnp_er40d_er40sadrt', 'cnp_pcet_pcetcr', 'cnp_pcet_pcetrtcr',
'cnp_pcet_pcet_eff', 'cnp_pcet_pcet_acc', 'cnp_pcet_pcet_acc2',
'cnp_medf36_medf36_hap_rtcr', 'cnp_medf36_medf36_sad_rtcr',
'cnp_medf36_medf36_ang_rtcr', 'cnp_medf36_medf36_fear_rtcr',
'cnp_medf36_medf36_a', 'cnp_medf36_medf36_t', 'cnp_pvoc_pvoccr',
'cnp_pvoc_pvocrtcr', 'cnp_pvrt_pvrtcr', 'cnp_pvrt_pvrtrtcr',
'cnp_pvrt_pvrt_eff', 'cnp_svdelay_svt_ld', 'cnp_svdelay_svtldrtc',
'cnp_svdelay_svtldrter', 'cnp_svdelay_svt_effd', 'cnp_sfnb2_sfnb_mrt',
'cnp_sfnb2_sfnb_meff', 'np_gpeg_dh_time', 'np_gpeg_ndh_time',
'np_wais4_rawscore', 'np_wrat4_wr_raw', 'np_wrat4_mc_raw',
'cnp_spcptnl_scpn_tp', 'cnp_spcptnl_scpn_tprt', 'cnp_spcptnl_scpl_tp',
'cnp_cpfd_dfac_tot', 'cnp_cpfd_dfac_rtc']]

final_df.to_csv('np_release.csv')
<|endoftext|>


<commit_before>import requests
from newspaper import Article
import os, datetime, time, tempfile, codecs, json, xlrd, logging
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
from flask import Response, abort
import csv
import docx2txt
from werkzeug.utils import secure_filename
from bs4 import UnicodeDammit

import databasic

logger = logging.getLogger(__name__)

ENCODING_UTF_8 = 'utf-8'
ENCODING_UTF_16 = 'utf-16'
ENCODING_LATIN_1 = 'latin-1'
TEMP_DIR = tempfile.gettempdir()

samples = []
docs = None
ACCEPTED_EXTENSIONS = ['txt', 'docx', 'rtf', 'csv', 'xlsx', 'xls']

def init_samples():
    global samples
    samples_config_file_path = os.path.join(databasic.get_config_dir(), 'sample-data.json')
    samples = json.load(open(samples_config_file_path))
    if databasic.app.config.get(databasic.ENV_APP_MODE) == databasic.APP_MODE_DEV:
        # change the paths to absolute onesls
        for sample in samples:
            sample['path'] = os.path.join(databasic.get_base_dir(), sample['source'])
        logger.info("Updated sample data with base dir: %s" % databasic.get_base_dir())
    else:
        # copy from server to local temp dir and change to abs paths (to temp dir files)
        url_base = databasic.app.config.get('SAMPLE_DATA_SERVER')
        for sample in samples:
            url = url_base + sample['source']
            logger.info("Loading sample data file: %s" % url)
            text = requests.get(url).text
            f = tempfile.NamedTemporaryFile(mode="w", delete=False)
            f.write(text)
            f.close()
            sample['path'] = f.name
        logger.info("Downloaded sample data and saved to tempdir")
    for sample in samples:
        file_size = os.stat(sample['path']).st_size
        logger.debug("  Cached %d bytes of %s to %s", file_size, sample['source'], sample['path'])

def write_to_temp_file(text):
    file_path = _get_temp_file()
    logger.debug("writing %d chars to %s" % (len(text), file_path))
    tmp_file = open(file_path, 'w')
    tmp_file.write(text)
    tmp_file.close()
    return file_path

def write_to_csv(headers, rows, file_name_suffix=None, timestamp=True):
    file_path = _get_temp_file(file_name_suffix, timestamp)
    with open(file_path, 'w') as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        for row in rows:
            writer.writerow(row)
    return file_path

def generate_csv(file_path):
    file_name = _get_file_name(file_path)
    if not os.path.isfile(file_path):
        return abort(400)

    def generate():
        with open(file_path, 'r') as f:
            reader = csv.reader(f)
            for row in reader:
                yield ','.join(row) + '\n'

    return Response(generate(), headers={'Content-Disposition': 'attachment;filename='+file_name}, mimetype='text/csv')

def convert_to_txt(file_path):
    logger.debug("convert_to_txt: %s" % file_path)
    if not os.path.exists(file_path):
        logger.error("missing file %s", file_path)
    file_size = os.stat(file_path).st_size
    logger.debug("convert_to_txt: %d bytes at %s", file_size, file_path)
    ext = _get_extension(file_path)
    if ext == '.txt':
        logger.debug("loading txt file")
        try:
            encoding, file_handle, words = open_with_correct_encoding(file_path)
        except Exception:
            logger.error("Wasn't able to read the words from the file %s" % file_path)
            words = ""
    elif ext == '.docx':
        logger.debug("loading docx file")
        words = _docx_to_txt(file_path)
    elif ext == '.rtf':
        logger.debug("loading rtf file")
        doc = Rtf15Reader.read(open(file_path))
        words = PlaintextWriter.write(doc).getvalue()
    else:
        logging.warning("Couldn't find an extension on the file, so assuming text")
        with open(file_path, 'r') as myfile:
            words = myfile.read()
    logger.debug("loaded %d chars" % len(words))
    return words

def convert_to_utf8(file_path):
    encoding, file_handle, content = open_with_correct_encoding(file_path)
    if encoding is ENCODING_UTF_8:
        return file_path
    # now we have to save it as utf8
    return write_to_temp_file(content)

def open_with_correct_encoding(file_path):
    """
    Since you can't actually detect the encoding of a file perfectly, default to utf-8
    and fallback to any others we want.  This returns the file handle, and the content, since
    you have to try to read the file for it to fail on a content encoding error.
    """
    file_handle = open(file_path, 'r')
    content = file_handle.read()
    dammit = UnicodeDammit(content)
    file_handle.seek(0)
    return [dammit.original_encoding, file_handle, content]

def convert_to_csv(file_path):
    ext = _get_extension(file_path)
    if ext == '.csv':
        return [file_path]
    elif ext == '.xlsx' or ext == '.xls':
        wb = xlrd.open_workbook(file_path)
        files = []
        for i in range(wb.nsheets):
            files.append(_open_sheet(wb, i))
        return files
    logger.error(ext + ' could not be converted to csv')
    return [file_path]

def open_doc(doc):
    filename = secure_filename(doc.filename)
    file_path = os.path.join(TEMP_DIR, filename)
    doc.save(file_path)
    return file_path

def open_docs(doc_list):
    file_paths = []
    for doc in doc_list:
        file_paths.append(open_doc(doc))
    return file_paths

def delete_files(file_paths):
    for f in file_paths:
        delete_file(f)

def delete_file(file_path):
    os.remove(file_path)

def open_workbook(book):
    file_paths = []
    for i, worksheet in enumerate(book.worksheets()):
        file_path = _get_temp_file('-' + worksheet.title + '.csv')
        with open(file_path, 'wb') as f:
            writer = csv.writer(f, delimiter=str(';'), quotechar=str('"'))
            writer.writerows(worksheet.get_all_values())
        file_paths.append(file_path)
    return file_paths

def get_samples(tool_id, lang, domain=None):
    matching_samples = []
    for sample in samples:
        if tool_id in sample['modules'] and sample['lang'] == lang:  # filter samples by language and tool
            if (domain is None) or ('domains' not in sample) or (domain in sample['domains']): # filter by domain (if specified)
                logger.debug("domain from browser is " + domain)
                if os.path.exists(sample['path']):
                    # only include samples we have been able to download from the static sample server URL
                    matching_samples.append((sample['source'], sample['title']))
                else:
                    logger.error("%s: file for %s doesn't exist at %s", tool_id, sample['source'], sample['path'])
    return matching_samples

def get_sample(source):
    for text in samples:
        if source in text['source']:
            return text
    return None

def get_sample_title(source):
    sample = get_sample(source)
    return source if sample is None else sample['title']

def get_sample_path(source):
    sample = get_sample(source)
    return source if sample is None else sample['path']

def get_file_names(file_paths):
    file_names = []
    for f in file_paths:
        file_names.append(_get_file_name(f))
    return file_names

def generate_filename(ext, suffix, *args):
    files = '-'.join(args) + '-' if len(args) > 0 else ''
    suffix = suffix + '-' if suffix is not None and suffix != '' else ''
    suffix = suffix.replace(' ', '-')
    ext = ext[1:] if '.' in ext[0] else ext
    return files + suffix + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '.' + ext

def download_webpage(url):
    article = Article(url)
    article.download()
    article.parse()
    return {'title': article.title, 'text': article.text}

def _open_sheet(workbook, index):
    sh = workbook.sheet_by_index(index)
    name = workbook.sheet_names()[index]
    new_file = _get_temp_file('-' + name + '.csv')
    with open(new_file, 'wb') as f:
        writer = csv.writer(f, delimiter=str(','), quotechar=str('"'))
        for row in range(sh.nrows):
            writer.writerow(sh.row_values(row))
    return new_file

def _get_file_name(file_path):
    return os.path.split(file_path)[1]

def _get_temp_file(file_name_suffix=None, timestamp=True):
    file_name = ''
    if timestamp:
        file_name = time.strftime("%Y%m%d-%H%M%S")
    if file_name_suffix is not None:
        file_name += file_name_suffix
    file_path = os.path.join(TEMP_DIR, file_name)
    logger.debug("new tempfile at %s", file_path)
    return file_path

def _get_extension(file_path):
    return os.path.splitext(file_path)[1]

def _docx_to_txt(file_path):
    return docx2txt.process(file_path).decode('utf-8')
<commit_msg>Fix word counter .docx uploads #393<commit_after>import requests
from newspaper import Article
import os, datetime, time, tempfile, codecs, json, xlrd, logging
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
from flask import Response, abort
import csv
import docx2txt
from werkzeug.utils import secure_filename
from bs4 import UnicodeDammit

import databasic

logger = logging.getLogger(__name__)

ENCODING_UTF_8 = 'utf-8'
ENCODING_UTF_16 = 'utf-16'
ENCODING_LATIN_1 = 'latin-1'
TEMP_DIR = tempfile.gettempdir()

samples = []
docs = None
ACCEPTED_EXTENSIONS = ['txt', 'docx', 'rtf', 'csv', 'xlsx', 'xls']

def init_samples():
    global samples
    samples_config_file_path = os.path.join(databasic.get_config_dir(), 'sample-data.json')
    samples = json.load(open(samples_config_file_path))
    if databasic.app.config.get(databasic.ENV_APP_MODE) == databasic.APP_MODE_DEV:
        # change the paths to absolute onesls
        for sample in samples:
            sample['path'] = os.path.join(databasic.get_base_dir(), sample['source'])
        logger.info("Updated sample data with base dir: %s" % databasic.get_base_dir())
    else:
        # copy from server to local temp dir and change to abs paths (to temp dir files)
        url_base = databasic.app.config.get('SAMPLE_DATA_SERVER')
        for sample in samples:
            url = url_base + sample['source']
            logger.info("Loading sample data file: %s" % url)
            text = requests.get(url).text
            f = tempfile.NamedTemporaryFile(mode="w", delete=False)
            f.write(text)
            f.close()
            sample['path'] = f.name
        logger.info("Downloaded sample data and saved to tempdir")
    for sample in samples:
        file_size = os.stat(sample['path']).st_size
        logger.debug("  Cached %d bytes of %s to %s", file_size, sample['source'], sample['path'])

def write_to_temp_file(text):
    file_path = _get_temp_file()
    logger.debug("writing %d chars to %s" % (len(text), file_path))
    tmp_file = open(file_path, 'w')
    tmp_file.write(text)
    tmp_file.close()
    return file_path

def write_to_csv(headers, rows, file_name_suffix=None, timestamp=True):
    file_path = _get_temp_file(file_name_suffix, timestamp)
    with open(file_path, 'w') as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        for row in rows:
            writer.writerow(row)
    return file_path

def generate_csv(file_path):
    file_name = _get_file_name(file_path)
    if not os.path.isfile(file_path):
        return abort(400)

    def generate():
        with open(file_path, 'r') as f:
            reader = csv.reader(f)
            for row in reader:
                yield ','.join(row) + '\n'

    return Response(generate(), headers={'Content-Disposition': 'attachment;filename='+file_name}, mimetype='text/csv')

def convert_to_txt(file_path):
    logger.debug("convert_to_txt: %s" % file_path)
    if not os.path.exists(file_path):
        logger.error("missing file %s", file_path)
    file_size = os.stat(file_path).st_size
    logger.debug("convert_to_txt: %d bytes at %s", file_size, file_path)
    ext = _get_extension(file_path)
    if ext == '.txt':
        logger.debug("loading txt file")
        try:
            encoding, file_handle, words = open_with_correct_encoding(file_path)
        except Exception:
            logger.error("Wasn't able to read the words from the file %s" % file_path)
            words = ""
    elif ext == '.docx':
        logger.debug("loading docx file")
        words = _docx_to_txt(file_path)
    elif ext == '.rtf':
        logger.debug("loading rtf file")
        doc = Rtf15Reader.read(open(file_path))
        words = PlaintextWriter.write(doc).getvalue()
    else:
        logging.warning("Couldn't find an extension on the file, so assuming text")
        with open(file_path, 'r') as myfile:
            words = myfile.read()
    logger.debug("loaded %d chars" % len(words))
    return words

def convert_to_utf8(file_path):
    encoding, file_handle, content = open_with_correct_encoding(file_path)
    if encoding is ENCODING_UTF_8:
        return file_path
    # now we have to save it as utf8
    return write_to_temp_file(content)

def open_with_correct_encoding(file_path):
    """
    Since you can't actually detect the encoding of a file perfectly, default to utf-8
    and fallback to any others we want.  This returns the file handle, and the content, since
    you have to try to read the file for it to fail on a content encoding error.
    """
    file_handle = open(file_path, 'r')
    content = file_handle.read()
    dammit = UnicodeDammit(content)
    file_handle.seek(0)
    return [dammit.original_encoding, file_handle, content]

def convert_to_csv(file_path):
    ext = _get_extension(file_path)
    if ext == '.csv':
        return [file_path]
    elif ext == '.xlsx' or ext == '.xls':
        wb = xlrd.open_workbook(file_path)
        files = []
        for i in range(wb.nsheets):
            files.append(_open_sheet(wb, i))
        return files
    logger.error(ext + ' could not be converted to csv')
    return [file_path]

def open_doc(doc):
    filename = secure_filename(doc.filename)
    file_path = os.path.join(TEMP_DIR, filename)
    doc.save(file_path)
    return file_path

def open_docs(doc_list):
    file_paths = []
    for doc in doc_list:
        file_paths.append(open_doc(doc))
    return file_paths

def delete_files(file_paths):
    for f in file_paths:
        delete_file(f)

def delete_file(file_path):
    os.remove(file_path)

def open_workbook(book):
    file_paths = []
    for i, worksheet in enumerate(book.worksheets()):
        file_path = _get_temp_file('-' + worksheet.title + '.csv')
        with open(file_path, 'wb') as f:
            writer = csv.writer(f, delimiter=str(';'), quotechar=str('"'))
            writer.writerows(worksheet.get_all_values())
        file_paths.append(file_path)
    return file_paths

def get_samples(tool_id, lang, domain=None):
    matching_samples = []
    for sample in samples:
        if tool_id in sample['modules'] and sample['lang'] == lang:  # filter samples by language and tool
            if (domain is None) or ('domains' not in sample) or (domain in sample['domains']): # filter by domain (if specified)
                logger.debug("domain from browser is " + domain)
                if os.path.exists(sample['path']):
                    # only include samples we have been able to download from the static sample server URL
                    matching_samples.append((sample['source'], sample['title']))
                else:
                    logger.error("%s: file for %s doesn't exist at %s", tool_id, sample['source'], sample['path'])
    return matching_samples

def get_sample(source):
    for text in samples:
        if source in text['source']:
            return text
    return None

def get_sample_title(source):
    sample = get_sample(source)
    return source if sample is None else sample['title']

def get_sample_path(source):
    sample = get_sample(source)
    return source if sample is None else sample['path']

def get_file_names(file_paths):
    file_names = []
    for f in file_paths:
        file_names.append(_get_file_name(f))
    return file_names

def generate_filename(ext, suffix, *args):
    files = '-'.join(args) + '-' if len(args) > 0 else ''
    suffix = suffix + '-' if suffix is not None and suffix != '' else ''
    suffix = suffix.replace(' ', '-')
    ext = ext[1:] if '.' in ext[0] else ext
    return files + suffix + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '.' + ext

def download_webpage(url):
    article = Article(url)
    article.download()
    article.parse()
    return {'title': article.title, 'text': article.text}

def _open_sheet(workbook, index):
    sh = workbook.sheet_by_index(index)
    name = workbook.sheet_names()[index]
    new_file = _get_temp_file('-' + name + '.csv')
    with open(new_file, 'wb') as f:
        writer = csv.writer(f, delimiter=str(','), quotechar=str('"'))
        for row in range(sh.nrows):
            writer.writerow(sh.row_values(row))
    return new_file

def _get_file_name(file_path):
    return os.path.split(file_path)[1]

def _get_temp_file(file_name_suffix=None, timestamp=True):
    file_name = ''
    if timestamp:
        file_name = time.strftime("%Y%m%d-%H%M%S")
    if file_name_suffix is not None:
        file_name += file_name_suffix
    file_path = os.path.join(TEMP_DIR, file_name)
    logger.debug("new tempfile at %s", file_path)
    return file_path

def _get_extension(file_path):
    return os.path.splitext(file_path)[1]

def _docx_to_txt(file_path):
    return docx2txt.process(file_path)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Sample conversion script for 1,000,000 VA data.

The 1,000,000 VA data is comprised of VCFs with every position
called.  This script converts those files to gVCF format.

Assumptions:
- one sample per input VCF

TODO(Cuiping and Denis):
 (1) Add logic for VCF block records such as depth and genotype quality.  For
     more detail, see https://sites.google.com/site/gvcftools/home/about-gvcf
     and http://www.broadinstitute.org/gatk/guide/article?id=4017
 (2) Use your favority python unit test framework to add some tests for the
     logic to collapse contiguous non-variant calls into a VCF block record.

This script can be run standalone:
   zcat chr21.vcf.gz | ./gvcf-mapper.py

Or via the debugger:
   python -mpdb ./gvcf-mapper.py chr1.vcf

It can also be run as a Hadoop Streaming job:
  hadoop jar /path/to/your/hadoop-streaming-*.jar \
  -libjars /home/deflaux/custom.jar \
  -outputformat com.custom.CustomMultiOutputFormat \
  -mapper /home/deflaux/gvcf-mapper.py -file /home/deflaux/gvcf-mapper.py \
  -reducer org.apache.hadoop.mapred.lib.IdentityReducer \
  -input inputpath \
  -output outputpath

See also https://developers.google.com/hadoop/

This is a modified version of the simple gvcf conversion script.  It uses the reduce step and a custom output format to consolidate data for each sample to a smaller number of files.  For more detail, see
    http://stackoverflow.com/questions/7449756/get-input-file-name-in-streaming-hadoop-program
    http://stackoverflow.com/questions/18541503/multiple-output-files-for-hadoop-streaming-with-python-mapper
"""

import os
import re
import sys
import pdb

# Constants
INPUT_FILE_KEY = "map_input_file"
SAMPLE_ID_PATTERN = r"/(LP\d{7}-DNA_\w\d{2})/"

# VCF Fields
# http://www.1000genomes.org/wiki/Analysis/Variant%20Call%20Format/vcf-variant-call-format-version-41
CHROM = 0
POS = 1
ID = 2
REF = 3
ALT = 4
QUAL = 5
FILTER = 6
INFO = 7
FORMAT = 8
GENOTYPE = 9

vcf_count = 0
filtered_count = 0
total_count = 0

# Using global variables in an effort to keep this script simple
g_start_block = None
g_end_block = None
min_qual = None
min_depth = None
min_mq = None
max_mq0 = None

def main():
  """Entry point to the script."""

  global sample_id
  sample_id = None
  sample_id_re = re.compile(SAMPLE_ID_PATTERN)

  # Basic parsing of command line arguments to allow a filename
  # to be passed when running this code in the debugger.
  file_handle = sys.stdin
  if 2 <= len(sys.argv):
    path = sys.argv[1]
    file_handle = open(path, "r")
  else:
    path = os.environ[INPUT_FILE_KEY]
    print >> sys.stderr, path

  match = sample_id_re.search(path)
  sample_id = match.group(1)

  # Loop over each line of the VCF
  line = file_handle.readline()
  while line:
    total_count =+ 1
    line = line.strip()
    if not line:
      continue

    if "#" == line[0]:
      # This is a header line, skip it
      line = file_handle.readline()
      continue

    fields = line.split("\t")
    if is_variant(fields):
        if meets_filter_criteria(fields) == True:
          # This is a variant, emit the preceeding non-variant region VCF block, if
          # applicable, followed by this VCF line
          emit_block(sample_id)
          emit(sample_id, line)
    else:
      # Gather information about this VCF line in our non-variant region
      if meets_filter_criteria(fields) == True:
        accumulate_block(fields)

    line = file_handle.readline()

  # Emit the final block, if applicable
  emit_block(sample_id)

def accumulate_block(fields):
  """Accumulates data from each record in a non-variant region.

  Every consecutive VCF record in a non-variant region is passed to
  this function.

  You might modify it to observe the minimum or average quality score
  for records in this region.

  You might also decide within this function that these fields should
  be part of a new VCF block record instead of the current one.  If
  so, call emit_block() and then begin accumulating state for the new
  VCF block record.

  Args:
    fields: (string array) fields from one line of the VCF

  Returns: n/a

  Side Effects:
    global variables are modified
  """
  global g_start_block
  global g_end_block
  global min_qual
  global min_depth
  global min_mq
  global max_mq0

  if g_start_block is None:
    g_start_block = fields
  elif g_end_block is not None and int(fields[POS]) > int(g_end_block[POS]) + 1:
      emit_block(sample_id)
      g_start_block = fields
  else:
    g_end_block = fields

  variant_info_dict = info_to_dict(fields)

  min_qual = check_values(min_qual, fields[QUAL])
  min_depth = check_values(min_depth, variant_info_dict['DP'])
  min_mq = check_values(min_mq, variant_info_dict['MQ'])
  max_mq0 = check_values(max_mq0, variant_info_dict['MQ0'], min=False)

def check_values(existing, new, min=True):
  if existing is None:
    existing = new
  else:
    if min is True:
      if existing > new:
        existing = new
    else:
      if existing < new:
        existing = new
  return existing

def emit_block(key):
  """Emits the current non-variant block, if applicable.

  The current implementation naively uses all values from the first
  VCF record in the block overwriting the INFO value with the end of
  the block.

  You might modify this function to utilize more accumulated state about
  the block such as minimum or average quality score.

  You might also merge the END INFO key/value pairs with any other
  existing INFO key/values.

  Args:
    key: (string) the key upon which to sort prior to the reduce step

  Returns
  : n/a

  Side Effects:
    global variables are modified
    a VCF line is written to stdout
  """
  global g_start_block
  global g_end_block
  global min_qual
  global min_depth
  global min_mq
  global max_mq0

  if g_start_block is None:
    return

  block_fields = g_start_block

  if g_end_block is None:
    block_fields[INFO] = "END=" + str(int(g_start_block[POS]) + 1)
  else:
    block_fields[INFO] = "END=" + g_end_block[POS]

  block_fields[INFO] += ";DP=" + min_depth
  block_fields[INFO] += ";MQ=" + min_mq
  block_fields[INFO] += ";MQ0=" + max_mq0
  block_fields[QUAL] = min_qual

  line = "\t".join(block_fields)
  emit(key, line)
  # Reset our block state
  g_start_block = None
  g_end_block = None
  min_depth = None
  min_qual = None
  min_mq = None
  max_mq0 = None

def emit(key, line):
  """Emits a VCF line to stdout.

  Args:
    key: (string) the key upon which to sort prior to the reduce step
    line: (string)

  Returns: n/a

  Side Effects:
    a VCF line is written to stdout
  """
  print "%s\t%s" % (key, line)

def is_variant(fields):
  """Determines whether or not the VCF fields constitute variant.

  Args:
    fields: (string array) fields from one line of the VCF

  Returns:
    (boolean) whether or not the VCF field constitute a variant

  Side Effects:
    a VCF line is written to stdout
  """
  if "0|0" in fields[GENOTYPE] or "0/0" in fields[GENOTYPE]:
    return False
  return True

def meets_filter_criteria(fields):
  """
   filter criteria 1: (returns true if)
   if ALT field is . (Mean a reference call) then
   check that the values if the INFO field meet the following requirements:
   1) MQ0 less than 4
   2) MQ greater than or = to 30
   3) QUAL greater than or = to 30
  """
  if fields[ALT] != ".":
    if fields[FILTER] == "PASS":
      return True
    else:
      return False
  if fields[ALT] == ".":    # Means it is a reference call Now to check the values meet our threshold

    variant_info_dict = info_to_dict(fields)

    ## Processing the above dict to meet criteria.
    if float(variant_info_dict['MQ0']) < 4 and float(variant_info_dict['MQ']) >= 30 and float(fields[QUAL]) >= 30:
      vcf_count =+ 1
      return True
    else:
      filtered_count =+ 1
      return False
  else:
    vcf_count =+ 1
    return True
  return True

def info_to_dict(fields):
  ##Preprocessing: Takes the string value of INFO column (field[INFO]) and converts into dict as Key, Value based on the =
  # info_string_items = [s for s in fields[INFO].split(';') if s]
  variant_info_dict = {}
  for item in info_string_items:
    try:
      key,value = item.split('=')
      variant_info_dict[key] = value
    except:
      pass  #Exception handling for keys without values, Specifically 'DB' in the INFO string
  return variant_info_dict

if __name__ == "__main__":
  main()
<commit_msg>Bug fix<commit_after>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Sample conversion script for 1,000,000 VA data.

The 1,000,000 VA data is comprised of VCFs with every position
called.  This script converts those files to gVCF format.

Assumptions:
- one sample per input VCF

TODO(Cuiping and Denis):
 (1) Add logic for VCF block records such as depth and genotype quality.  For
     more detail, see https://sites.google.com/site/gvcftools/home/about-gvcf
     and http://www.broadinstitute.org/gatk/guide/article?id=4017
 (2) Use your favority python unit test framework to add some tests for the
     logic to collapse contiguous non-variant calls into a VCF block record.

This script can be run standalone:
   zcat chr21.vcf.gz | ./gvcf-mapper.py

Or via the debugger:
   python -mpdb ./gvcf-mapper.py chr1.vcf

It can also be run as a Hadoop Streaming job:
  hadoop jar /path/to/your/hadoop-streaming-*.jar \
  -libjars /home/deflaux/custom.jar \
  -outputformat com.custom.CustomMultiOutputFormat \
  -mapper /home/deflaux/gvcf-mapper.py -file /home/deflaux/gvcf-mapper.py \
  -reducer org.apache.hadoop.mapred.lib.IdentityReducer \
  -input inputpath \
  -output outputpath

See also https://developers.google.com/hadoop/

This is a modified version of the simple gvcf conversion script.  It uses the reduce step and a custom output format to consolidate data for each sample to a smaller number of files.  For more detail, see
    http://stackoverflow.com/questions/7449756/get-input-file-name-in-streaming-hadoop-program
    http://stackoverflow.com/questions/18541503/multiple-output-files-for-hadoop-streaming-with-python-mapper
"""

import os
import re
import sys
import pdb

# Constants
INPUT_FILE_KEY = "map_input_file"
SAMPLE_ID_PATTERN = r"/(LP\d{7}-DNA_\w\d{2})/"

# VCF Fields
# http://www.1000genomes.org/wiki/Analysis/Variant%20Call%20Format/vcf-variant-call-format-version-41
CHROM = 0
POS = 1
ID = 2
REF = 3
ALT = 4
QUAL = 5
FILTER = 6
INFO = 7
FORMAT = 8
GENOTYPE = 9

vcf_count = 0
filtered_count = 0
total_count = 0

# Using global variables in an effort to keep this script simple
g_start_block = None
g_end_block = None
min_qual = None
min_depth = None
min_mq = None
max_mq0 = None

def main():
  """Entry point to the script."""

  global sample_id
  sample_id = None
  sample_id_re = re.compile(SAMPLE_ID_PATTERN)

  # Basic parsing of command line arguments to allow a filename
  # to be passed when running this code in the debugger.
  file_handle = sys.stdin
  if 2 <= len(sys.argv):
    path = sys.argv[1]
    file_handle = open(path, "r")
  else:
    path = os.environ[INPUT_FILE_KEY]
    print >> sys.stderr, path

  match = sample_id_re.search(path)
  sample_id = match.group(1)

  # Loop over each line of the VCF
  line = file_handle.readline()
  while line:
    total_count =+ 1
    line = line.strip()
    if not line:
      continue

    if "#" == line[0]:
      # This is a header line, skip it
      line = file_handle.readline()
      continue

    fields = line.split("\t")
    if is_variant(fields):
        if meets_filter_criteria(fields) == True:
          # This is a variant, emit the preceeding non-variant region VCF block, if
          # applicable, followed by this VCF line
          emit_block(sample_id)
          emit(sample_id, line)
    else:
      # Gather information about this VCF line in our non-variant region
      if meets_filter_criteria(fields) == True:
        accumulate_block(fields)

    line = file_handle.readline()

  # Emit the final block, if applicable
  emit_block(sample_id)

def accumulate_block(fields):
  """Accumulates data from each record in a non-variant region.

  Every consecutive VCF record in a non-variant region is passed to
  this function.

  You might modify it to observe the minimum or average quality score
  for records in this region.

  You might also decide within this function that these fields should
  be part of a new VCF block record instead of the current one.  If
  so, call emit_block() and then begin accumulating state for the new
  VCF block record.

  Args:
    fields: (string array) fields from one line of the VCF

  Returns: n/a

  Side Effects:
    global variables are modified
  """
  global g_start_block
  global g_end_block
  global min_qual
  global min_depth
  global min_mq
  global max_mq0

  if g_start_block is None:
    g_start_block = fields
  elif g_end_block is not None and int(fields[POS]) > int(g_end_block[POS]) + 1:
      emit_block(sample_id)
      g_start_block = fields
  else:
    g_end_block = fields

  variant_info_dict = info_to_dict(fields)

  min_qual = check_values(min_qual, fields[QUAL])
  min_depth = check_values(min_depth, variant_info_dict['DP'])
  min_mq = check_values(min_mq, variant_info_dict['MQ'])
  max_mq0 = check_values(max_mq0, variant_info_dict['MQ0'], min=False)

def check_values(existing, new, min=True):
  if existing is None:
    existing = new
  else:
    if min is True:
      if existing > new:
        existing = new
    else:
      if existing < new:
        existing = new
  return existing

def emit_block(key):
  """Emits the current non-variant block, if applicable.

  The current implementation naively uses all values from the first
  VCF record in the block overwriting the INFO value with the end of
  the block.

  You might modify this function to utilize more accumulated state about
  the block such as minimum or average quality score.

  You might also merge the END INFO key/value pairs with any other
  existing INFO key/values.

  Args:
    key: (string) the key upon which to sort prior to the reduce step

  Returns
  : n/a

  Side Effects:
    global variables are modified
    a VCF line is written to stdout
  """
  global g_start_block
  global g_end_block
  global min_qual
  global min_depth
  global min_mq
  global max_mq0

  if g_start_block is None:
    return

  block_fields = g_start_block

  if g_end_block is None:
    block_fields[INFO] = "END=" + str(int(g_start_block[POS]) + 1)
  else:
    block_fields[INFO] = "END=" + g_end_block[POS]

  block_fields[INFO] += ";DP=" + min_depth
  block_fields[INFO] += ";MQ=" + min_mq
  block_fields[INFO] += ";MQ0=" + max_mq0
  block_fields[QUAL] = min_qual

  line = "\t".join(block_fields)
  emit(key, line)
  # Reset our block state
  g_start_block = None
  g_end_block = None
  min_depth = None
  min_qual = None
  min_mq = None
  max_mq0 = None

def emit(key, line):
  """Emits a VCF line to stdout.

  Args:
    key: (string) the key upon which to sort prior to the reduce step
    line: (string)

  Returns: n/a

  Side Effects:
    a VCF line is written to stdout
  """
  print "%s\t%s" % (key, line)

def is_variant(fields):
  """Determines whether or not the VCF fields constitute variant.

  Args:
    fields: (string array) fields from one line of the VCF

  Returns:
    (boolean) whether or not the VCF field constitute a variant

  Side Effects:
    a VCF line is written to stdout
  """
  if "0|0" in fields[GENOTYPE] or "0/0" in fields[GENOTYPE]:
    return False
  return True

def meets_filter_criteria(fields):
  """
   filter criteria 1: (returns true if)
   if ALT field is . (Mean a reference call) then
   check that the values if the INFO field meet the following requirements:
   1) MQ0 less than 4
   2) MQ greater than or = to 30
   3) QUAL greater than or = to 30
  """
  if fields[ALT] != ".":
    if fields[FILTER] == "PASS":
      return True
    else:
      return False
  if fields[ALT] == ".":    # Means it is a reference call Now to check the values meet our threshold

    variant_info_dict = info_to_dict(fields)

    ## Processing the above dict to meet criteria.
    if float(variant_info_dict['MQ0']) < 4 and float(variant_info_dict['MQ']) >= 30 and float(fields[QUAL]) >= 30:
      vcf_count =+ 1
      return True
    else:
      filtered_count =+ 1
      return False
  else:
    vcf_count =+ 1
    return True
  return True

def info_to_dict(fields):
  ##Preprocessing: Takes the string value of INFO column (field[INFO]) and converts into dict as Key, Value based on the =
  info_string_items = [s for s in fields[INFO].split(';') if s]
  variant_info_dict = {}
  for item in info_string_items:
    try:
      key,value = item.split('=')
      variant_info_dict[key] = value
    except:
      pass  #Exception handling for keys without values, Specifically 'DB' in the INFO string
  return variant_info_dict

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import traceback
import argparse
from PIL import Image

def in_rect(x, y, rect):
    if x >= rect[0][0] and y>=rect[0][1] and x < rect[1][0] and y < rect[1][1]:
        return True
    return False

def string_reverse(s):
    if s == '0':
        return '1'
    return '0'

def binterpret(filename, abx=8, aby=8, offsetx=0, offsety=0, msizex=8, msizey=8, inverse=False):
    try:
        img = Image.open(filename)
    except IOError:
        traceback.print_exc(file=sys.stdout)
        exit(3)
    blockx = (img.size[0] - offsetx)/abx
    blocky = (img.size[1] - offsety - 25)/aby

    binary_data = ""

    for y in range(0, aby):
        for x in range(0, abx):

            #The three markings in the corner
            if in_rect(x, y, [(0,0), (markx,marky)]):
                continue
            if in_rect(x, y, [(abx - markx, 0), (abx,marky)]):
                continue
            if in_rect(x, y, [(0,aby - marky), (markx,aby)]):
                continue

            to_add = '1' if not inverse else '0'
            pixel_red = img.getpixel((offsetx + x * blockx, offsety + y * blocky))[0]
            new_data = to_add if pixel_red < 128 else string_reverse(to_add)
            binary_data += new_data

    return binary_data

if __name__ == "__main__":
    DEFAULT = 8
    parser = argparse.ArgumentParser(description='Read a QRcode as binary data')

    #Converting arguments
    parser.add_argument('filename', help="The image to interpret")
    parser.add_argument('-xblocks', type=int, help="The amount of squares in width.  Default is 8")
    parser.add_argument('-yblocks', type=int, help="The amount of squares in height. Default is 8")
    parser.add_argument('-offsetx', type=int, help="The x-offset in pixels")
    parser.add_argument('-offsety', type=int, help="The y-offset in pixels")
    parser.add_argument('-markx', type=int, help="The amount of squares of the markers in width. Default is 8.")
    parser.add_argument('-marky', type=int, help="The amount of squares of the markers in height. Default is 8.")
    parser.add_argument('--inverse', action='store_true', default=False, help="Inverse the binary data")

    #Flag arguments
    parser.add_argument('--ascii', action='store_true', default=False, help="Print the binary data as ascii")
        #Flag arguments
    parser.add_argument('--binary', action='store_true', default=False, help="Print the binary data as binary")

    args = parser.parse_args()

    xblocks = args.xblocks if args.xblocks else DEFAULT
    yblocks = args.yblocks if args.yblocks else DEFAULT
    offsetx = args.offsetx if args.offsetx else 0
    offsety = args.offsety if args.offsety else 0
    markx = args.markx if args.markx else DEFAULT
    marky = args.marky if args.marky else DEFAULT

    data = binterpret(args.filename, xblocks, yblocks, offsetx, offsety, markx, marky, args.inverse)

    if args.binary:
        print data

    if args.ascii:
        d = [data[8*i:8*(i+1)] for i in range(len(data)/8)]
        d = [int(i, 2) for i in d]
        print "".join(chr(i) for i in d)
<commit_msg>Added executable permission<commit_after>#!/usr/bin/env python

import sys
import traceback
import argparse
from PIL import Image

def in_rect(x, y, rect):
    if x >= rect[0][0] and y>=rect[0][1] and x < rect[1][0] and y < rect[1][1]:
        return True
    return False

def string_reverse(s):
    if s == '0':
        return '1'
    return '0'

def binterpret(filename, abx=8, aby=8, offsetx=0, offsety=0, msizex=8, msizey=8, inverse=False):
    try:
        img = Image.open(filename)
    except IOError:
        traceback.print_exc(file=sys.stdout)
        exit(3)
    blockx = (img.size[0] - offsetx)/abx
    blocky = (img.size[1] - offsety - 25)/aby

    binary_data = ""

    for y in range(0, aby):
        for x in range(0, abx):

            #The three markings in the corner
            if in_rect(x, y, [(0,0), (markx,marky)]):
                continue
            if in_rect(x, y, [(abx - markx, 0), (abx,marky)]):
                continue
            if in_rect(x, y, [(0,aby - marky), (markx,aby)]):
                continue

            to_add = '1' if not inverse else '0'
            pixel_red = img.getpixel((offsetx + x * blockx, offsety + y * blocky))[0]
            new_data = to_add if pixel_red < 128 else string_reverse(to_add)
            binary_data += new_data

    return binary_data

if __name__ == "__main__":
    DEFAULT = 8
    parser = argparse.ArgumentParser(description='Read a QRcode as binary data')

    #Converting arguments
    parser.add_argument('filename', help="The image to interpret")
    parser.add_argument('-xblocks', type=int, help="The amount of squares in width.  Default is 8")
    parser.add_argument('-yblocks', type=int, help="The amount of squares in height. Default is 8")
    parser.add_argument('-offsetx', type=int, help="The x-offset in pixels")
    parser.add_argument('-offsety', type=int, help="The y-offset in pixels")
    parser.add_argument('-markx', type=int, help="The amount of squares of the markers in width. Default is 8.")
    parser.add_argument('-marky', type=int, help="The amount of squares of the markers in height. Default is 8.")
    parser.add_argument('--inverse', action='store_true', default=False, help="Inverse the binary data")

    #Flag arguments
    parser.add_argument('--ascii', action='store_true', default=False, help="Print the binary data as ascii")
        #Flag arguments
    parser.add_argument('--binary', action='store_true', default=False, help="Print the binary data as binary")

    args = parser.parse_args()

    xblocks = args.xblocks if args.xblocks else DEFAULT
    yblocks = args.yblocks if args.yblocks else DEFAULT
    offsetx = args.offsetx if args.offsetx else 0
    offsety = args.offsety if args.offsety else 0
    markx = args.markx if args.markx else DEFAULT
    marky = args.marky if args.marky else DEFAULT

    data = binterpret(args.filename, xblocks, yblocks, offsetx, offsety, markx, marky, args.inverse)

    if args.binary:
        print data

    if args.ascii:
        d = [data[8*i:8*(i+1)] for i in range(len(data)/8)]
        d = [int(i, 2) for i in d]
        print "".join(chr(i) for i in d)
<|endoftext|>