<commit_before>"""
Ripped off from waf (v 1.6), by Thomas Nagy

The cool design is his, bugs most certainly mine :)
"""

import os, shutil, re, sys

def split_path(path):
    return path.split('/')

def split_path_cygwin(path):
    if path.startswith('//'):
        ret = path.split('/')[2:]
        ret[0] = '/' + ret[0]
        return ret
    return path.split('/')

re_sp = re.compile('[/\\\\]')
def split_path_win32(path):
    if path.startswith('\\\\'):
        ret = re.split(re_sp, path)[2:]
        ret[0] = '\\' + ret[0]
        return ret
    return re.split(re_sp, path)

if sys.platform == 'cygwin':
    split_path = split_path_cygwin
elif sys.platform == 'win32':
    split_path = split_path_win32

class Node(object):
    """
    This class is divided into two parts, the basic methods meant for filesystem access, and
    the ones which are bound to a context (the build context). The imports do not reflect this.
    """

    __slots__ = ('name', 'sig', 'children', 'parent', 'cache_abspath', 'cache_isdir')
    def __init__(self, name, parent):
        import inspect
        #print inspect.stack()[3], name, parent
        self.name = name
        self.parent = parent

        if parent:
            if name in parent.children:
                raise Errors.WafError('node %s exists in the parent files %r already' % (name, parent))
            parent.children[name] = self

    def __setstate__(self, data):
        self.name = data[0]
        self.parent = data[1]
        if data[2] is not None:
            self.children = data[2]
        if data[3] is not None:
            self.sig = data[3]

    def __getstate__(self):
        return (self.name, self.parent, getattr(self, 'children', None), getattr(self, 'sig', None))

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.abspath()

    def __hash__(self):
        return id(self) # TODO see if it is still the case
        #raise Errors.WafError('do not hash nodes (too expensive)')

    def __eq__(self, node):
        return id(self) == id(node)

    def __copy__(self):
        "nodes are not supposed to be copied"
        raise Errors.WafError('nodes are not supposed to be copied')

    def read(self, flags='r'):
        "get the contents, assuming the node is a file"
        fid = open(self.abspath(), flags)
        try:
            return fid.read()
        finally:
            fid.close()

    def write(self, data, flags='w'):
        "write some text to the physical file, assuming the node is a file"
        f = None
        try:
            f = open(self.abspath(), flags)
            f.write(data)
        finally:
            if f:
                f.close()

    def chmod(self, val):
        "change file/dir permissions"
        os.chmod(self.abspath(), val)

    def delete(self):
        "delete the file physically, do not destroy the nodes"
        try:
            shutil.rmtree(self.abspath())
        except:
            pass

        try:
            delattr(self, 'children')
        except:
            pass

    def suffix(self):
        "scons-like - hot zone so do not touch"
        k = max(0, self.name.rfind('.'))
        return self.name[k:]

    def height(self):
        "amount of parents"
        d = self
        val = -1
        while d:
            d = d.parent
            val += 1
        return val

    def listdir(self):
        "list the directory contents"
        return Utils.listdir(self.abspath())

    def mkdir(self):
        "write a directory for the node"
        if getattr(self, 'cache_isdir', None):
            return

        try:
            self.parent.mkdir()
        except:
            pass

        if self.name:
            try:
                os.mkdir(self.abspath())
            except OSError as e:
                pass

            if not os.path.isdir(self.abspath()):
                raise Errors.WafError('%s is not a directory' % self)

            try:
                self.children
            except:
                self.children = {}

        self.cache_isdir = True

    def find_node(self, lst):
        "read the file system, make the nodes as needed"

        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        cur = self
        for x in lst:
            if x == '..':
                cur = cur.parent
                continue

            try:
                if x in cur.children:
                    cur = cur.children[x]
                    continue
            except:
                cur.children = {}

            # optimistic: create the node first then look if it was correct to do so
            cur = self.__class__(x, cur)
            try:
                os.stat(cur.abspath())
            except:
                del cur.parent.children[x]
                return None

        ret = cur

        try:
            while not getattr(cur.parent, 'cache_isdir', None):
                cur = cur.parent
                cur.cache_isdir = True
        except AttributeError:
            pass

        return ret

    def make_node(self, lst):
        "make a branch of nodes"
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        cur = self
        for x in lst:
            if x == '..':
                cur = cur.parent
                continue

            if getattr(cur, 'children', {}):
                if x in cur.children:
                    cur = cur.children[x]
                    continue
            else:
                cur.children = {}
            cur = self.__class__(x, cur)
        return cur

    def search(self, lst):
        "dumb search for existing nodes"
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        cur = self
        try:
            for x in lst:
                if x == '..':
                    cur = cur.parent
                else:
                    cur = cur.children[x]
            return cur
        except:
            pass

    def path_from(self, node):
        """path of this node seen from the other
            self = foo/bar/xyz.txt
            node = foo/stuff/
            -> ../bar/xyz.txt
        """
        # common root in rev 7673

        c1 = self
        c2 = node

        c1h = c1.height()
        c2h = c2.height()

        lst = []
        up = 0

        while c1h > c2h:
            lst.append(c1.name)
            c1 = c1.parent
            c1h -= 1

        while c2h > c1h:
            up += 1
            c2 = c2.parent
            c2h -= 1

        while id(c1) != id(c2):
            lst.append(c1.name)
            up += 1

            c1 = c1.parent
            c2 = c2.parent

        for i in range(up):
            lst.append('..')
        lst.reverse()
        return os.sep.join(lst) or '.'

    def abspath(self):
        """
        absolute path
        cache into the build context, cache_node_abspath
        """
        try:
            return self.cache_abspath
        except:
            pass
        # think twice before touching this (performance + complexity + correctness)
        if not self.parent:
            val = os.sep == '/' and os.sep or ''
        elif not self.parent.name:
            # drive letter for win32
            val = (os.sep == '/' and os.sep or '') + self.name
        else:
            val = self.parent.abspath() + os.sep + self.name

        self.cache_abspath = val
        return val

    def is_child_of(self, node):
        "does this node belong to the subtree node"
        p = self
        diff = self.height() - node.height()
        while diff > 0:
            diff -= 1
            p = p.parent
        return id(p) == id(node)

    # --------------------------------------------------------------------------------
    # the following methods require the source/build folders (bld.srcnode/bld.bldnode)
    # using a subclass is a possibility, but is that really necessary?
    # --------------------------------------------------------------------------------

    def is_src(self):
        """note: !is_src does not imply is_bld()"""
        cur = self
        x = id(self.ctx.srcnode)
        y = id(self.ctx.bldnode)
        while cur.parent:
            if id(cur) == y:
                return False
            if id(cur) == x:
                return True
            cur = cur.parent
        return False

    def is_bld(self):
        """note: !is_bld does not imply is_src"""
        cur = self
        y = id(self.ctx.bldnode)
        while cur.parent:
            if id(cur) == y:
                return True
            cur = cur.parent
        return False

    def get_src(self):
        """for a build node, will return the equivalent src node (or self if not possible)"""
        cur = self
        x = id(self.ctx.srcnode)
        y = id(self.ctx.bldnode)
        lst = []
        while cur.parent:
            if id(cur) == y:
                lst.reverse()
                return self.ctx.srcnode.make_node(lst)
            if id(cur) == x:
                return self
            lst.append(cur.name)
            cur = cur.parent
        return self

    def get_bld(self):
        """for a src node, will return the equivalent bld node (or self if not possible)"""
        cur = self
        x = id(self.ctx.srcnode)
        y = id(self.ctx.bldnode)
        lst = []
        while cur.parent:
            if id(cur) == y:
                return self
            if id(cur) == x:
                lst.reverse()
                return self.ctx.bldnode.make_node(lst)
            lst.append(cur.name)
            cur = cur.parent
        return self

    def find_resource(self, lst):
        """
        try to find a declared build node or a source file
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']
        elif issubclass(type(lst), type(self)):
            return lst

        node = self.get_bld().search(lst)
        if node:
            return node

        self = self.get_src()
        node = self.search(lst)
        if node:
            return node

        node = self.find_node(lst)
        if node:
            return node

        return node

    def find_or_declare(self, lst):
        """
        if 'self' is in build directory, try to return an existing node
        if no node is found, go to the source directory
        try to find an existing node in the source directory
        if no node is found, create it in the build directory
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        node = self.get_bld().search(lst)
        if node:
            if not os.path.isfile(node.abspath()):
                node.sig = None
                try:
                    node.parent.mkdir()
                except:
                    pass
            return node
        self = self.get_src()
        node = self.find_node(lst)
        if node:
            if not os.path.isfile(node.abspath()):
                node.sig = None
                try:
                    node.parent.mkdir()
                except:
                    pass
            return node
        node = self.get_bld().make_node(lst)
        node.parent.mkdir()
        return node

    def declare(self, lst):
        """
        if 'self' is in build directory, try to return an existing node
        if no node is found, create it in the build directory
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        node = self.get_bld().search(lst)
        if node:
            if not os.path.isfile(node.abspath()):
                node.sig = None
                try:
                    node.parent.mkdir()
                except:
                    pass
            return node
        node = self.get_bld().make_node(lst)
        node.parent.mkdir()
        return node

    def find_dir(self, lst):
        """
        search a folder in the filesystem
        create the corresponding mappings source <-> build directories
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        node = self.find_node(lst)
        try:
            os.path.isdir(node.abspath())
        except OSError:
            return None
        return node

    # helpers for building things
    def change_ext(self, ext):
        "node of the same path, but with a different extension - hot zone so do not touch"
        name = self.name
        # XXX: is using name.find(".") as done in waf a bug ?
        k = name.rfind('.')
        if k >= 0:
            name = name[:k] + ext
        else:
            name = name + ext

        return self.parent.find_or_declare([name])

    def nice_path(self, env=None):
        "printed in the console, open files easily from the launch directory"
        return self.path_from(self.ctx.launch_node())

    def bldpath(self):
        "path seen from the build directory default/src/foo.cpp"
        return self.path_from(self.ctx.bldnode)

    def srcpath(self):
        "path seen from the source directory ../src/foo.cpp"
        return self.path_from(self.ctx.srcnode)

    def relpath(self):
        "if a build node, bldpath, else srcpath"
        cur = self
        x = id(self.ctx.bldnode)
        while cur.parent:
            if id(cur) == x:
                return self.ctxpath()
            cur = cur.parent
        return self.srcpath()

    def bld_dir(self):
        "build path without the file name"
        return self.parent.bldpath()

    def bld_base(self):
        "build path without the extension: src/dir/foo(.cpp)"
        s = os.path.splitext(self.name)[0]
        return self.bld_dir() + os.sep + s
<commit_msg>BUG: remove waflib utils usage.<commit_after>"""
Ripped off from waf (v 1.6), by Thomas Nagy

The cool design is his, bugs most certainly mine :)
"""

import os, shutil, re, sys

def split_path(path):
    return path.split('/')

def split_path_cygwin(path):
    if path.startswith('//'):
        ret = path.split('/')[2:]
        ret[0] = '/' + ret[0]
        return ret
    return path.split('/')

re_sp = re.compile('[/\\\\]')
def split_path_win32(path):
    if path.startswith('\\\\'):
        ret = re.split(re_sp, path)[2:]
        ret[0] = '\\' + ret[0]
        return ret
    return re.split(re_sp, path)

if sys.platform == 'cygwin':
    split_path = split_path_cygwin
elif sys.platform == 'win32':
    split_path = split_path_win32

class Node(object):
    """
    This class is divided into two parts, the basic methods meant for filesystem access, and
    the ones which are bound to a context (the build context). The imports do not reflect this.
    """

    __slots__ = ('name', 'sig', 'children', 'parent', 'cache_abspath', 'cache_isdir')
    def __init__(self, name, parent):
        import inspect
        #print inspect.stack()[3], name, parent
        self.name = name
        self.parent = parent

        if parent:
            if name in parent.children:
                raise Errors.WafError('node %s exists in the parent files %r already' % (name, parent))
            parent.children[name] = self

    def __setstate__(self, data):
        self.name = data[0]
        self.parent = data[1]
        if data[2] is not None:
            self.children = data[2]
        if data[3] is not None:
            self.sig = data[3]

    def __getstate__(self):
        return (self.name, self.parent, getattr(self, 'children', None), getattr(self, 'sig', None))

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.abspath()

    def __hash__(self):
        return id(self) # TODO see if it is still the case
        #raise Errors.WafError('do not hash nodes (too expensive)')

    def __eq__(self, node):
        return id(self) == id(node)

    def __copy__(self):
        "nodes are not supposed to be copied"
        raise Errors.WafError('nodes are not supposed to be copied')

    def read(self, flags='r'):
        "get the contents, assuming the node is a file"
        fid = open(self.abspath(), flags)
        try:
            return fid.read()
        finally:
            fid.close()

    def write(self, data, flags='w'):
        "write some text to the physical file, assuming the node is a file"
        f = None
        try:
            f = open(self.abspath(), flags)
            f.write(data)
        finally:
            if f:
                f.close()

    def chmod(self, val):
        "change file/dir permissions"
        os.chmod(self.abspath(), val)

    def delete(self):
        "delete the file physically, do not destroy the nodes"
        try:
            shutil.rmtree(self.abspath())
        except:
            pass

        try:
            delattr(self, 'children')
        except:
            pass

    def suffix(self):
        "scons-like - hot zone so do not touch"
        k = max(0, self.name.rfind('.'))
        return self.name[k:]

    def height(self):
        "amount of parents"
        d = self
        val = -1
        while d:
            d = d.parent
            val += 1
        return val

    def listdir(self):
        "list the directory contents"
        return os.listdir(self.abspath())

    def mkdir(self):
        "write a directory for the node"
        if getattr(self, 'cache_isdir', None):
            return

        try:
            self.parent.mkdir()
        except:
            pass

        if self.name:
            try:
                os.mkdir(self.abspath())
            except OSError as e:
                pass

            if not os.path.isdir(self.abspath()):
                raise Errors.WafError('%s is not a directory' % self)

            try:
                self.children
            except:
                self.children = {}

        self.cache_isdir = True

    def find_node(self, lst):
        "read the file system, make the nodes as needed"

        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        cur = self
        for x in lst:
            if x == '..':
                cur = cur.parent
                continue

            try:
                if x in cur.children:
                    cur = cur.children[x]
                    continue
            except:
                cur.children = {}

            # optimistic: create the node first then look if it was correct to do so
            cur = self.__class__(x, cur)
            try:
                os.stat(cur.abspath())
            except:
                del cur.parent.children[x]
                return None

        ret = cur

        try:
            while not getattr(cur.parent, 'cache_isdir', None):
                cur = cur.parent
                cur.cache_isdir = True
        except AttributeError:
            pass

        return ret

    def make_node(self, lst):
        "make a branch of nodes"
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        cur = self
        for x in lst:
            if x == '..':
                cur = cur.parent
                continue

            if getattr(cur, 'children', {}):
                if x in cur.children:
                    cur = cur.children[x]
                    continue
            else:
                cur.children = {}
            cur = self.__class__(x, cur)
        return cur

    def search(self, lst):
        "dumb search for existing nodes"
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        cur = self
        try:
            for x in lst:
                if x == '..':
                    cur = cur.parent
                else:
                    cur = cur.children[x]
            return cur
        except:
            pass

    def path_from(self, node):
        """path of this node seen from the other
            self = foo/bar/xyz.txt
            node = foo/stuff/
            -> ../bar/xyz.txt
        """
        # common root in rev 7673

        c1 = self
        c2 = node

        c1h = c1.height()
        c2h = c2.height()

        lst = []
        up = 0

        while c1h > c2h:
            lst.append(c1.name)
            c1 = c1.parent
            c1h -= 1

        while c2h > c1h:
            up += 1
            c2 = c2.parent
            c2h -= 1

        while id(c1) != id(c2):
            lst.append(c1.name)
            up += 1

            c1 = c1.parent
            c2 = c2.parent

        for i in range(up):
            lst.append('..')
        lst.reverse()
        return os.sep.join(lst) or '.'

    def abspath(self):
        """
        absolute path
        cache into the build context, cache_node_abspath
        """
        try:
            return self.cache_abspath
        except:
            pass
        # think twice before touching this (performance + complexity + correctness)
        if not self.parent:
            val = os.sep == '/' and os.sep or ''
        elif not self.parent.name:
            # drive letter for win32
            val = (os.sep == '/' and os.sep or '') + self.name
        else:
            val = self.parent.abspath() + os.sep + self.name

        self.cache_abspath = val
        return val

    def is_child_of(self, node):
        "does this node belong to the subtree node"
        p = self
        diff = self.height() - node.height()
        while diff > 0:
            diff -= 1
            p = p.parent
        return id(p) == id(node)

    # --------------------------------------------------------------------------------
    # the following methods require the source/build folders (bld.srcnode/bld.bldnode)
    # using a subclass is a possibility, but is that really necessary?
    # --------------------------------------------------------------------------------

    def is_src(self):
        """note: !is_src does not imply is_bld()"""
        cur = self
        x = id(self.ctx.srcnode)
        y = id(self.ctx.bldnode)
        while cur.parent:
            if id(cur) == y:
                return False
            if id(cur) == x:
                return True
            cur = cur.parent
        return False

    def is_bld(self):
        """note: !is_bld does not imply is_src"""
        cur = self
        y = id(self.ctx.bldnode)
        while cur.parent:
            if id(cur) == y:
                return True
            cur = cur.parent
        return False

    def get_src(self):
        """for a build node, will return the equivalent src node (or self if not possible)"""
        cur = self
        x = id(self.ctx.srcnode)
        y = id(self.ctx.bldnode)
        lst = []
        while cur.parent:
            if id(cur) == y:
                lst.reverse()
                return self.ctx.srcnode.make_node(lst)
            if id(cur) == x:
                return self
            lst.append(cur.name)
            cur = cur.parent
        return self

    def get_bld(self):
        """for a src node, will return the equivalent bld node (or self if not possible)"""
        cur = self
        x = id(self.ctx.srcnode)
        y = id(self.ctx.bldnode)
        lst = []
        while cur.parent:
            if id(cur) == y:
                return self
            if id(cur) == x:
                lst.reverse()
                return self.ctx.bldnode.make_node(lst)
            lst.append(cur.name)
            cur = cur.parent
        return self

    def find_resource(self, lst):
        """
        try to find a declared build node or a source file
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']
        elif issubclass(type(lst), type(self)):
            return lst

        node = self.get_bld().search(lst)
        if node:
            return node

        self = self.get_src()
        node = self.search(lst)
        if node:
            return node

        node = self.find_node(lst)
        if node:
            return node

        return node

    def find_or_declare(self, lst):
        """
        if 'self' is in build directory, try to return an existing node
        if no node is found, go to the source directory
        try to find an existing node in the source directory
        if no node is found, create it in the build directory
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        node = self.get_bld().search(lst)
        if node:
            if not os.path.isfile(node.abspath()):
                node.sig = None
                try:
                    node.parent.mkdir()
                except:
                    pass
            return node
        self = self.get_src()
        node = self.find_node(lst)
        if node:
            if not os.path.isfile(node.abspath()):
                node.sig = None
                try:
                    node.parent.mkdir()
                except:
                    pass
            return node
        node = self.get_bld().make_node(lst)
        node.parent.mkdir()
        return node

    def declare(self, lst):
        """
        if 'self' is in build directory, try to return an existing node
        if no node is found, create it in the build directory
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        node = self.get_bld().search(lst)
        if node:
            if not os.path.isfile(node.abspath()):
                node.sig = None
                try:
                    node.parent.mkdir()
                except:
                    pass
            return node
        node = self.get_bld().make_node(lst)
        node.parent.mkdir()
        return node

    def find_dir(self, lst):
        """
        search a folder in the filesystem
        create the corresponding mappings source <-> build directories
        """
        if isinstance(lst, str):
            lst = [x for x in split_path(lst) if x and x != '.']

        node = self.find_node(lst)
        try:
            os.path.isdir(node.abspath())
        except OSError:
            return None
        return node

    # helpers for building things
    def change_ext(self, ext):
        "node of the same path, but with a different extension - hot zone so do not touch"
        name = self.name
        # XXX: is using name.find(".") as done in waf a bug ?
        k = name.rfind('.')
        if k >= 0:
            name = name[:k] + ext
        else:
            name = name + ext

        return self.parent.find_or_declare([name])

    def nice_path(self, env=None):
        "printed in the console, open files easily from the launch directory"
        return self.path_from(self.ctx.launch_node())

    def bldpath(self):
        "path seen from the build directory default/src/foo.cpp"
        return self.path_from(self.ctx.bldnode)

    def srcpath(self):
        "path seen from the source directory ../src/foo.cpp"
        return self.path_from(self.ctx.srcnode)

    def relpath(self):
        "if a build node, bldpath, else srcpath"
        cur = self
        x = id(self.ctx.bldnode)
        while cur.parent:
            if id(cur) == x:
                return self.ctxpath()
            cur = cur.parent
        return self.srcpath()

    def bld_dir(self):
        "build path without the file name"
        return self.parent.bldpath()

    def bld_base(self):
        "build path without the extension: src/dir/foo(.cpp)"
        s = os.path.splitext(self.name)[0]
        return self.bld_dir() + os.sep + s
<|endoftext|>


<commit_before>from __future__ import print_function, division
import signal, sys, os

import click

from .version import __version__

@click.group()
@click.version_option(version=__version__)
def cli():
    # to make this script/module behave nicely with unix pipes
    # http://newbebweb.blogspot.com/2012/02/python-head-ioerror-errno-32-broken.html
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

@cli.command()
@click.option('--workspace', required=True, type=click.Path(),
              help='A directory to place outputs into')
@click.option('--job-db', default=None, type=click.Path(),
              help="Path to LSF job sqlite DB [default='<workspace>/job_queue.db']")
@click.option('--input-vcfs', required=True, type=click.Path(exists=True),
              help='A file of chromosomal VCFs to process')
@click.option('--project-name', default='yaps2.default', type=click.STRING,
              help='A prefix used to name batch jobs')
@click.option('--email', default=None, type=click.STRING,
              help='An email used to notify about batch jobs [default=userid@genome.wustl.edu]')
@click.option('--drm', default='lsf', type=click.Choice(['local', 'lsf']),
              help='Job Mode -- [default=lsf]')
@click.option('--restart/--no-restart', default=False,),
              help='Restart Pipeline from scratch')
def postvqsr(job_db, input_vcfs, project_name, email, workspace, drm, restart):
    from yap2.pipelines.postvqsr import Config, Pipeline
    config = Config(job_db, input_vcfs, project_name, email, workspace)
    workflow = Pipeline(config, drm, restart)
    workflow.run()
<commit_msg>+ remove incorrect syntax parentheses<commit_after>from __future__ import print_function, division
import signal, sys, os

import click

from .version import __version__

@click.group()
@click.version_option(version=__version__)
def cli():
    # to make this script/module behave nicely with unix pipes
    # http://newbebweb.blogspot.com/2012/02/python-head-ioerror-errno-32-broken.html
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

@cli.command()
@click.option('--workspace', required=True, type=click.Path(),
              help='A directory to place outputs into')
@click.option('--job-db', default=None, type=click.Path(),
              help="Path to LSF job sqlite DB [default='<workspace>/job_queue.db']")
@click.option('--input-vcfs', required=True, type=click.Path(exists=True),
              help='A file of chromosomal VCFs to process')
@click.option('--project-name', default='yaps2.default', type=click.STRING,
              help='A prefix used to name batch jobs')
@click.option('--email', default=None, type=click.STRING,
              help='An email used to notify about batch jobs [default=userid@genome.wustl.edu]')
@click.option('--drm', default='lsf', type=click.Choice(['local', 'lsf']),
              help='Job Mode -- [default=lsf]')
@click.option('--restart/--no-restart', default=False,
              help='Restart Pipeline from scratch')
def postvqsr(job_db, input_vcfs, project_name, email, workspace, drm, restart):
    from yap2.pipelines.postvqsr import Config, Pipeline
    config = Config(job_db, input_vcfs, project_name, email, workspace)
    workflow = Pipeline(config, drm, restart)
    workflow.run()
<|endoftext|>


<commit_before>from __future__ import print_function, division
import signal, sys, os

import click

from .version import __version__

@click.group()
@click.version_option(version=__version__)
def cli():
    # to make this script/module behave nicely with unix pipes
    # http://newbebweb.blogspot.com/2012/02/python-head-ioerror-errno-32-broken.html
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

@cli.command()
@click.option('--workspace', required=True, type=click.Path(),
              help='A directory to place outputs into')
@click.option('--job-db', default=None, type=click.Path(),
              help="Path to LSF job sqlite DB [default='<workspace>/job_queue.db']")
@click.option('--input-vcfs', required=True, type=click.Path(exists=True),
              help='A file of chromosomal VCFs to process')
@click.option('--project-name', default='yaps2.default', type=click.STRING,
              help='A prefix used to name batch jobs')
@click.option('--email', default=None, type=click.STRING,
              help='An email used to notify about batch jobs [default=userid@genome.wustl.edu]')
@click.option('--drm', default='lsf', type=click.Choice(['local', 'lsf']),
              help='Job Mode -- [default=lsf]')
@click.option('--restart/--no-restart', default=False,
              help='Restart Pipeline from scratch')
def postvqsr(job_db, input_vcfs, project_name, email, workspace, drm, restart):
    from yaps2.pipelines.postvqsr import Config, Pipeline
    config = Config(job_db, input_vcfs, project_name, email, workspace)
    workflow = Pipeline(config, drm, restart)
    workflow.run()

@cli.command()
@click.option('--workspace', required=True, type=click.Path(),
              help='A directory to place outputs into')
@click.option('--job-db', default=None, type=click.Path(),
              help="Path to LSF job sqlite DB [default='<workspace>/job_queue.db']")
@click.option('--input-vcfs', required=True, type=click.Path(exists=True),
              help='A file of chromosomal VCFs to process')
@click.option('--samples', required=True, type=click.Path(exists=True),
              help='A list of samples to restrict to [sample-name delimited by newlines]')
@click.option('--plink-fam', required=True, type=click.Path(exists=True),
              help='A plink .fam file describing the sample relationships')
@click.option('--percentiles', required=True, type=click.Path(exists=True),
              help='A tsv of category/percentile/min-vqslod/max-vqslod lines')
@click.option('--tranches', required=True, type=click.Path(exists=True),
              help='A tsv of category/tranche/min-vqslod lines')
@click.option('--project-name', default='yaps2.default', type=click.STRING,
              help='A prefix used to name batch jobs')
@click.option('--email', default=None, type=click.STRING,
              help='An email used to notify about batch jobs [default=userid@genome.wustl.edu]')
@click.option('--drm', default='lsf', type=click.Choice(['local', 'lsf']),
              help='Job Mode -- [default=lsf]')
@click.option('--restart/--no-restart', default=False,
              help='Restart Pipeline from scratch')
def mie(job_db, input_vcfs, percentiles, samples, tranches, plink_fam,
        project_name, email, workspace, drm, restart):
    from yaps2.pipelines.mie import Config, Pipeline
    config = Config(
        job_db, input_vcfs,
        percentiles, samples, tranches, plink_fam,
        project_name, email, workspace
    )
    workflow = Pipeline(config, drm, restart)
    workflow.run()
<commit_msg>+ introduce a '.' prefix to the default sqlite3 db file name<commit_after>from __future__ import print_function, division
import signal, sys, os

import click

from .version import __version__

@click.group()
@click.version_option(version=__version__)
def cli():
    # to make this script/module behave nicely with unix pipes
    # http://newbebweb.blogspot.com/2012/02/python-head-ioerror-errno-32-broken.html
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

@cli.command()
@click.option('--workspace', required=True, type=click.Path(),
              help='A directory to place outputs into')
@click.option('--job-db', default=None, type=click.Path(),
              help="Path to LSF job sqlite DB [default='<workspace>/.job_queue.db']")
@click.option('--input-vcfs', required=True, type=click.Path(exists=True),
              help='A file of chromosomal VCFs to process')
@click.option('--project-name', default='yaps2.default', type=click.STRING,
              help='A prefix used to name batch jobs')
@click.option('--email', default=None, type=click.STRING,
              help='An email used to notify about batch jobs [default=userid@genome.wustl.edu]')
@click.option('--drm', default='lsf', type=click.Choice(['local', 'lsf']),
              help='Job Mode -- [default=lsf]')
@click.option('--restart/--no-restart', default=False,
              help='Restart Pipeline from scratch')
def postvqsr(job_db, input_vcfs, project_name, email, workspace, drm, restart):
    from yaps2.pipelines.postvqsr import Config, Pipeline
    config = Config(job_db, input_vcfs, project_name, email, workspace)
    workflow = Pipeline(config, drm, restart)
    workflow.run()

@cli.command()
@click.option('--workspace', required=True, type=click.Path(),
              help='A directory to place outputs into')
@click.option('--job-db', default=None, type=click.Path(),
              help="Path to LSF job sqlite DB [default='<workspace>/.job_queue.db']")
@click.option('--input-vcfs', required=True, type=click.Path(exists=True),
              help='A file of chromosomal VCFs to process')
@click.option('--samples', required=True, type=click.Path(exists=True),
              help='A list of samples to restrict to [sample-name delimited by newlines]')
@click.option('--plink-fam', required=True, type=click.Path(exists=True),
              help='A plink .fam file describing the sample relationships')
@click.option('--percentiles', required=True, type=click.Path(exists=True),
              help='A tsv of category/percentile/min-vqslod/max-vqslod lines')
@click.option('--tranches', required=True, type=click.Path(exists=True),
              help='A tsv of category/tranche/min-vqslod lines')
@click.option('--project-name', default='yaps2.default', type=click.STRING,
              help='A prefix used to name batch jobs')
@click.option('--email', default=None, type=click.STRING,
              help='An email used to notify about batch jobs [default=userid@genome.wustl.edu]')
@click.option('--drm', default='lsf', type=click.Choice(['local', 'lsf']),
              help='Job Mode -- [default=lsf]')
@click.option('--restart/--no-restart', default=False,
              help='Restart Pipeline from scratch')
def mie(job_db, input_vcfs, percentiles, samples, tranches, plink_fam,
        project_name, email, workspace, drm, restart):
    from yaps2.pipelines.mie import Config, Pipeline
    config = Config(
        job_db, input_vcfs,
        percentiles, samples, tranches, plink_fam,
        project_name, email, workspace
    )
    workflow = Pipeline(config, drm, restart)
    workflow.run()
<|endoftext|>


<commit_before>import os
import asyncio
from asyncio import sleep
import logging
from subprocess import check_output, call, check_call

from . import systemd

class AptPostgresqlPlugin:
    """Plugin for controlling postgresql installed by apt.

    NOTE: if you want to set the options in postgresql.conf, edit /etc/postgresql-common/createcluster.conf
    """

    def __init__(self, name, app):
        self.app = app
        self._health_check_key = (name, 'systemd')
        self.logger = logging.getLogger(name)

    @property
    def _version(self):
        return self.app.config['apt']['postgresql_version']

    @property
    def _cluster_name(self):
        return self.app.config['apt']['postgresql_cluster_name']

    def _config_file(self):
        return '/etc/postgresql/{}/{}/postgresql.conf'.format(self._version, self._cluster_name)

    def _data_dir(self):
        # TODO: we should look in the config file for this
        return '/var/lib/postgresql/{}/{}/'.format(self._version, self._cluster_name)

    def _service(self):
        return 'postgresql@{}-{}.service'.format(self._version, self._cluster_name)

    def postgresql_get_database_identifier(self):
        if not os.path.exists(self._config_file()):
            return None
        data = check_output([
            '/usr/lib/postgresql/{}/bin/pg_controldata'.format(self._version),
            self._data_dir()])
        for line in data.splitlines():
            if line.startswith('Data page checksum version:'):
                _, dbid = line.split(':', 1)
                dbid = dbid.strip()
                return dbid
        return None

    def postgresql_start(self):
        check_call(['systemctl', 'start', self._service()])

    def postgresql_stop(self):
        check_call(['systemctl', 'stop', self._service()])

    def postgresql_initdb(self):
        if os.path.exists(self._config_file()):
            check_call(['pg_dropcluster', '--stop', self._version, self._cluster_name])
        check_call(['pg_createcluster', self._version, self._cluster_name])

    def postgresql_am_i_replica(self):
        return os.path.exists(os.path.join(self._data_dir(), 'recovery.conf'))

    def start_monitoring(self):
        self.app.unhealthy(self._health_check_key, 'Waiting for first systemd check')
        loop = asyncio.get_event_loop()
        loop.call_soon(loop.create_task, self._monitor_systemd())

    async def _monitor_systemd(self):
        loop = asyncio.get_event_loop()
        while True:
            await sleep(1)
            status = call(['systemctl', 'is-active', self._service()])
            if status == 0:
                self.app.healthy(self._health_check_key)
            else:
                await sleep(2)
                status = call(['systemctl', 'is-active', self._service()])
                if status != 0:
                    self.app.unhealthy(self._health_check_key, 'inactive according to systemd')
<commit_msg>encoding<commit_after>import os
import asyncio
from asyncio import sleep
import logging
from subprocess import check_output, call, check_call

from . import systemd

class AptPostgresqlPlugin:
    """Plugin for controlling postgresql installed by apt.

    NOTE: if you want to set the options in postgresql.conf, edit /etc/postgresql-common/createcluster.conf
    """

    def __init__(self, name, app):
        self.app = app
        self._health_check_key = (name, 'systemd')
        self.logger = logging.getLogger(name)

    @property
    def _version(self):
        return self.app.config['apt']['postgresql_version']

    @property
    def _cluster_name(self):
        return self.app.config['apt']['postgresql_cluster_name']

    def _config_file(self):
        return '/etc/postgresql/{}/{}/postgresql.conf'.format(self._version, self._cluster_name)

    def _data_dir(self):
        # TODO: we should look in the config file for this
        return '/var/lib/postgresql/{}/{}/'.format(self._version, self._cluster_name)

    def _service(self):
        return 'postgresql@{}-{}.service'.format(self._version, self._cluster_name)

    def postgresql_get_database_identifier(self):
        if not os.path.exists(self._config_file()):
            return None
        data = check_output([
            '/usr/lib/postgresql/{}/bin/pg_controldata'.format(self._version),
            self._data_dir()])
        data = data.decode('latin-1') # I don't care, don't fail, the data I am interested in is ascii
        for line in data.splitlines():
            if line.startswith('Data page checksum version:'):
                _, dbid = line.split(':', 1)
                dbid = dbid.strip()
                return dbid
        return None

    def postgresql_start(self):
        check_call(['systemctl', 'start', self._service()])

    def postgresql_stop(self):
        check_call(['systemctl', 'stop', self._service()])

    def postgresql_initdb(self):
        if os.path.exists(self._config_file()):
            check_call(['pg_dropcluster', '--stop', self._version, self._cluster_name])
        check_call(['pg_createcluster', self._version, self._cluster_name])

    def postgresql_am_i_replica(self):
        return os.path.exists(os.path.join(self._data_dir(), 'recovery.conf'))

    def start_monitoring(self):
        self.app.unhealthy(self._health_check_key, 'Waiting for first systemd check')
        loop = asyncio.get_event_loop()
        loop.call_soon(loop.create_task, self._monitor_systemd())

    async def _monitor_systemd(self):
        loop = asyncio.get_event_loop()
        while True:
            await sleep(1)
            status = call(['systemctl', 'is-active', self._service()])
            if status == 0:
                self.app.healthy(self._health_check_key)
            else:
                await sleep(2)
                status = call(['systemctl', 'is-active', self._service()])
                if status != 0:
                    self.app.unhealthy(self._health_check_key, 'inactive according to systemd')
<|endoftext|>


<commit_before>import os
import time
import shutil
import asyncio
from asyncio import sleep
import logging
from subprocess import check_output, call, check_call

import psycopg2
import psycopg2.errorcodes

from . import systemd, utils
from .plugin import subscribe

def _pg_controldata_value(pg_version, data_dir, key):
    data = check_output([
        '/usr/lib/postgresql/{}/bin/pg_controldata'.format(pg_version),
        data_dir])
    data = data.decode('latin-1') # I don't care, don't fail, the data I am interested in is ascii
    for line in data.splitlines():
        k, value = line.split(':', 1)
        if k == key:
            return value.strip()
    raise ValueError('could not find value for "{}"'.format(key))

class _NoCluster(Exception):
    pass

class AptPostgresqlPlugin:
    """Plugin for controlling postgresql installed by apt.

    NOTE: if you want to set the options in postgresql.conf, edit /etc/postgresql-common/createcluster.conf
    """

    def __init__(self, name, app):
        self.app = app
        self._health_check_key = '{}-systemd'.format(name)
        self.logger = logging.getLogger(name)

    @property
    def _create_superuser(self):
        must_create = self.app.config['apt'].get('create_superuser', '').lower().strip() in ('t', 'true')
        if must_create and self._superuser_connect_as is None:
            raise ValueError('must set superuser_connect_as if create_superuser is True')
        return must_create

    @property
    def _superuser_connect_as(self):
        val = self.app.config['apt'].get('superuser_connect_as')
        if val is not None:
            val = val.strip()
        return val

    @property
    def _version(self):
        return self.app.config['apt']['postgresql_version']

    @property
    def _cluster_name(self):
        return self.app.config['apt']['postgresql_cluster_name']

    @property
    def _config_dir(self):
        return self.app.config['apt'].get('config_dir')

    def _pg_config_dir(self):
        return '/etc/postgresql/{}/{}/'.format(self._version, self._cluster_name)

    def _config_file(self, name='postgresql.conf'):
        return os.path.join(self._pg_config_dir(), name)

    def _set_conf_value(self, key, value):
        value = check_call(['pg_conftool', self._version, self._cluster_name, 'set', key, value])

    def _get_conf_value(self, key):
        if not os.path.exists(self._config_file()):
            raise _NoCluster()
        value = check_output(['pg_conftool', '-s', self._version, self._cluster_name, 'show', key])
        value = value.decode('ascii').strip() # encoding unspecified, ascii is safe...
        if value.startswith("'") and value.endswith("'"):
            value = value[1:-1]
        return value.replace("''", "'").replace("\\'", "'")

    def _port(self):
        return self._get_conf_value('port')

    def _socket_dir(self):
        return self._get_conf_value('unix_socket_directories').split(',')[0]

    def _data_dir(self):
        return self._get_conf_value('data_directory')

    def _service(self):
        return 'postgresql@{}-{}.service'.format(self._version, self._cluster_name)

    def _copy_config(self):
        if not self._config_dir:
            return
        for filename in ['environment', 'pg_ctl.conf', 'pg_hba.conf', 'pg_ident.conf', 'postgresql.conf', 'start.conf']:
            source = os.path.join(self._config_dir, filename)
            if not os.path.exists(source):
                # no source, so don't replace target
                continue
            destination = self._config_file(filename)
            shutil.copyfile(source, destination)

    def _set_config_values(self, prefix=None):
        changed = False
        for k, v in self.app.config['apt'].items():
            if prefix is not None:
                if k.startswith(prefix):
                    k = k[len(prefix):]
                else:
                    continue
            if k.startswith('postgresql.conf.'):
                k = k[16:]
                v = v.strip()
                changed = True
                self._set_conf_value(k, v)
        changed = self._twiddle_config_file('pg_hba.conf') or changed
        changed = self._twiddle_config_file('pg_ident.conf') or changed
        return changed

    def _twiddle_config_file(self, config_file):
        prefix = config_file + '.'
        prefix_len = len(prefix)
        lines = {}
        for k, v in self.app.config['apt'].items():
            if k.startswith(prefix):
                k = k[prefix_len:]
                assert k not in lines
                lines[k] = v
        if not lines:
            return False
        mode = self.app.config['apt'].get(config_file, 'append')
        to_write = ['# {} by zgres'.format(mode)]
        for key, lines in sorted(lines.items()):
            to_write.append('')
            to_write.append('# added by zgres ({})'.format(key))
            to_write.append(lines)
        to_write.append('')
        path = self._config_file(name=config_file)
        if mode == 'append':
            with open(path, 'r') as f:
                currdata = f.read()
            to_write = [currdata, ''] + to_write
        elif mode == 'prepend':
            with open(path, 'r') as f:
                currdata = f.read()
            to_write = to_write + [currdata, '']
        elif mode == 'replace':
            pass
        else:
            raise ValueError(mode)
        with open(path, 'w') as f:
            currdata = f.write('\n'.join(to_write))
        return True

    @subscribe
    def pg_get_database_identifier(self):
        if not os.path.exists(self._config_file()):
            return None
        return _pg_controldata_value(self._version, self._data_dir(), 'Database system identifier')

    def _conn(self):
        info = self.pg_connect_info()
        return psycopg2.connect(**info)

    @subscribe
    def pg_get_timeline(self):
        if not os.path.exists(self._config_file()):
            return None
        if self._is_active():
            conn = self._conn()
            cur = conn.cursor()
            cur.execute('SELECT pg_xlogfile_name(pg_current_xlog_insert_location());')
            wal_filename = cur.fetchall()[0][0]
            timeline_hex = wal_filename[:8]
            return int(timeline_hex, 16)
        else:
            val = _pg_controldata_value(
                    self._version,
                    self._data_dir(),
                    "Latest checkpoint's TimeLineID")
            return int(val)
        return int(val)

    def _pg_accepts_connections(self):
        retval = call(['sudo', '-u', 'postgres', 'psql', '-h', self._socket_dir(), '-p', self._port(), '-c', 'SELECT 1'])
        if retval:
            return False
        return True

    def _wait_for_connections(self):
        utils.backoff_wait(self._pg_accepts_connections, 0.1, times=300, message='Waiting for postgresql to accept connections')

    @subscribe
    def pg_start(self):
        # TODO: implement pg_hba.conf reconfig to allow cluster nodes to join the cluster automatically
        try:
            check_call(['systemctl', 'start', self._service()])
        except Exception:
            call(['systemctl', 'status', self._service()])
            raise
        self._wait_for_connections()
        if not self._is_active():
            raise Exception('Failed to start postgresql')

    @subscribe
    def pg_reload(self):
        check_call(['systemctl', 'reload', self._service()])

    @subscribe
    def pg_stop(self):
        check_call(['systemctl', 'stop', self._service()])

    @subscribe
    def pg_reset(self):
        self.pg_stop()
        try:
            check_call(['pg_dropcluster', '--stop', self._version, self._cluster_name])
        except:
            if os.path.exists(self._config_file()):
                check_call(['rm', '-rf', self._data_dir(), self._config_dir])

    @subscribe
    def pg_initdb(self):
        if os.path.exists(self._config_file()):
            self.pg_stop()
            check_call(['pg_dropcluster', '--stop', self._version, self._cluster_name])
        create_args = ['pg_createcluster', self._version, self._cluster_name]
        initdb_options = self.app.config['apt'].get('initdb_options', '').strip()
        if initdb_options:
            initdb_options = [i.strip() for i in initdb_options.split() if i.strip()]
            create_args.append('--')
            create_args.extend(initdb_options)
        check_call(['pg_createcluster', self._version, self._cluster_name, '--', '--data-checksums'])
        self._copy_config()
        self._set_config_values()
        if self._create_superuser:
            self.pg_start()
            check_call(['sudo', '-u', 'postgres', 'createuser', '-s', '-h', self._socket_dir(), '-p', self._port(), self._superuser_connect_as])
            self.pg_stop()

    @subscribe
    def pg_connect_info(self):
        info = dict(database='postgres', user=self._superuser_connect_as, host=self._socket_dir(), port=self._port())
        user = self._superuser_connect_as
        if user is not None:
            info['user'] = user
        return info

    @subscribe
    def get_conn_info(self):
        return dict(port=self._port())

    @subscribe
    def pg_replication_role(self):
        try:
            return os.path.exists(os.path.join(self._data_dir(), 'recovery.conf')) and 'replica' or 'master'
        except _NoCluster:
            return None

    @subscribe
    def start_monitoring(self):
        self.app.unhealthy(self._health_check_key, 'Waiting for first systemd check')
        loop = asyncio.get_event_loop()
        loop.call_soon(loop.create_task, self._monitor_systemd())

    def _is_active(self):
        return 0 == call(['systemctl', '--quiet', 'is-active', self._service()])

    async def _monitor_systemd(self):
        loop = asyncio.get_event_loop()
        while True:
            await sleep(1)
            if self._is_active():
                self.app.healthy(self._health_check_key)
            else:
                await sleep(2)
                if not self._is_active():
                    self.app.unhealthy(self._health_check_key, 'inactive according to systemd')

    def _trigger_file(self):
        return '/var/run/postgresql/{}-{}.master_trigger'.format(self._version, self._cluster_name)

    def _pg_is_in_recovery(self):
        conn = self._conn()
        try:
            cur = conn.cursor()
            cur.execute('SELECT pg_is_in_recovery();')
            return cur.fetchall()[0][0]
        finally:
            conn.close()

    @subscribe
    def pg_stop_replication(self):
        assert self.pg_replication_role() == 'replica'
        trigger_file = self._trigger_file()
        with open(trigger_file, 'w') as f:
            f.write('touched')
        time.sleep(0.2)
        count = 0
        while count < 60:
            try:
                if not self._pg_is_in_recovery():
                    break
            except psycopg2.OperationalError as e:
                pass
            logging.info('waiting for postgresql to come out of recovery')
            count += 1
            time.sleep(1)
        else:
            # hmm, what to do here?
            raise Exception('postgresql did not come out of recovery mode')
        if self._set_config_values('master.'):
            self.pg_reload()

    @subscribe
    def pg_setup_replication(self, primary_conninfo=None):
        trigger_file = self._trigger_file()
        if os.path.exists(trigger_file):
            os.remove(trigger_file)
        config = """standby_mode = 'on'
trigger_file = '{trigger_file}'
recovery_target_timeline = 'latest'
"""
        config = config.format(trigger_file=trigger_file)
        restore_command = self.app.config['apt'].get('restore_command', None)
        if primary_conninfo:
            parts = []
            for k, v in primary_conninfo.items():
                parts.append('{}={}'.format(k, v))
            config += "\nprimary_conninfo = '{}'".format(' '.join(parts))
        if restore_command:
            config += "\nrestore_command = '{}'".format(restore_command)
        with open(os.path.join(self._data_dir(), 'recovery.conf'), 'w') as f:
            f.write(config)
        self._set_config_values('replica.')
<commit_msg>fix test<commit_after>import os
import time
import shutil
import asyncio
from asyncio import sleep
import logging
from subprocess import check_output, call, check_call

import psycopg2
import psycopg2.errorcodes

from . import systemd, utils
from .plugin import subscribe

def _pg_controldata_value(pg_version, data_dir, key):
    data = check_output([
        '/usr/lib/postgresql/{}/bin/pg_controldata'.format(pg_version),
        data_dir])
    data = data.decode('latin-1') # I don't care, don't fail, the data I am interested in is ascii
    for line in data.splitlines():
        k, value = line.split(':', 1)
        if k == key:
            return value.strip()
    raise ValueError('could not find value for "{}"'.format(key))

class _NoCluster(Exception):
    pass

class AptPostgresqlPlugin:
    """Plugin for controlling postgresql installed by apt.

    NOTE: if you want to set the options in postgresql.conf, edit /etc/postgresql-common/createcluster.conf
    """

    def __init__(self, name, app):
        self.app = app
        self._health_check_key = '{}-systemd'.format(name)
        self.logger = logging.getLogger(name)

    @property
    def _create_superuser(self):
        must_create = self.app.config['apt'].get('create_superuser', '').lower().strip() in ('t', 'true')
        if must_create and self._superuser_connect_as is None:
            raise ValueError('must set superuser_connect_as if create_superuser is True')
        return must_create

    @property
    def _superuser_connect_as(self):
        val = self.app.config['apt'].get('superuser_connect_as')
        if val is not None:
            val = val.strip()
        return val

    @property
    def _version(self):
        return self.app.config['apt']['postgresql_version']

    @property
    def _cluster_name(self):
        return self.app.config['apt']['postgresql_cluster_name']

    @property
    def _config_dir(self):
        return self.app.config['apt'].get('config_dir')

    def _pg_config_dir(self):
        return '/etc/postgresql/{}/{}/'.format(self._version, self._cluster_name)

    def _config_file(self, name='postgresql.conf'):
        return os.path.join(self._pg_config_dir(), name)

    def _set_conf_value(self, key, value):
        value = check_call(['pg_conftool', self._version, self._cluster_name, 'set', key, value])

    def _get_conf_value(self, key):
        if not os.path.exists(self._config_file()):
            raise _NoCluster()
        value = check_output(['pg_conftool', '-s', self._version, self._cluster_name, 'show', key])
        value = value.decode('ascii').strip() # encoding unspecified, ascii is safe...
        if value.startswith("'") and value.endswith("'"):
            value = value[1:-1]
        return value.replace("''", "'").replace("\\'", "'")

    def _port(self):
        return self._get_conf_value('port')

    def _socket_dir(self):
        return self._get_conf_value('unix_socket_directories').split(',')[0]

    def _data_dir(self):
        return self._get_conf_value('data_directory')

    def _service(self):
        return 'postgresql@{}-{}.service'.format(self._version, self._cluster_name)

    def _copy_config(self):
        if not self._config_dir:
            return
        for filename in ['environment', 'pg_ctl.conf', 'pg_hba.conf', 'pg_ident.conf', 'postgresql.conf', 'start.conf']:
            source = os.path.join(self._config_dir, filename)
            if not os.path.exists(source):
                # no source, so don't replace target
                continue
            destination = self._config_file(filename)
            shutil.copyfile(source, destination)

    def _set_config_values(self, prefix=None):
        changed = False
        for k, v in self.app.config['apt'].items():
            if prefix is not None:
                if k.startswith(prefix):
                    k = k[len(prefix):]
                else:
                    continue
            if k.startswith('postgresql.conf.'):
                k = k[16:]
                v = v.strip()
                changed = True
                self._set_conf_value(k, v)
        changed = self._twiddle_config_file('pg_hba.conf') or changed
        changed = self._twiddle_config_file('pg_ident.conf') or changed
        return changed

    def _twiddle_config_file(self, config_file):
        prefix = config_file + '.'
        prefix_len = len(prefix)
        lines = {}
        for k, v in self.app.config['apt'].items():
            if k.startswith(prefix):
                k = k[prefix_len:]
                assert k not in lines
                lines[k] = v
        if not lines:
            return False
        mode = self.app.config['apt'].get(config_file, 'append')
        to_write = ['# {} by zgres'.format(mode)]
        for key, lines in sorted(lines.items()):
            to_write.append('')
            to_write.append('# added by zgres ({})'.format(key))
            to_write.append(lines)
        to_write.append('')
        path = self._config_file(name=config_file)
        if mode == 'append':
            with open(path, 'r') as f:
                currdata = f.read()
            to_write = [currdata, ''] + to_write
        elif mode == 'prepend':
            with open(path, 'r') as f:
                currdata = f.read()
            to_write = to_write + [currdata, '']
        elif mode == 'replace':
            pass
        else:
            raise ValueError(mode)
        with open(path, 'w') as f:
            currdata = f.write('\n'.join(to_write))
        return True

    @subscribe
    def pg_get_database_identifier(self):
        if not os.path.exists(self._config_file()):
            return None
        return _pg_controldata_value(self._version, self._data_dir(), 'Database system identifier')

    def _conn(self):
        info = self.pg_connect_info()
        return psycopg2.connect(**info)

    @subscribe
    def pg_get_timeline(self):
        if not os.path.exists(self._config_file()):
            return None
        if self._is_active() and not self._pg_is_in_recovery():
            conn = self._conn()
            cur = conn.cursor()
            cur.execute('SELECT pg_xlogfile_name(pg_current_xlog_insert_location());')
            wal_filename = cur.fetchall()[0][0]
            timeline_hex = wal_filename[:8]
            return int(timeline_hex, 16)
        else:
            val = _pg_controldata_value(
                    self._version,
                    self._data_dir(),
                    "Latest checkpoint's TimeLineID")
            return int(val)

    def _pg_accepts_connections(self):
        retval = call(['sudo', '-u', 'postgres', 'psql', '-h', self._socket_dir(), '-p', self._port(), '-c', 'SELECT 1'])
        if retval:
            return False
        return True

    def _wait_for_connections(self):
        utils.backoff_wait(self._pg_accepts_connections, 0.1, times=300, message='Waiting for postgresql to accept connections')

    @subscribe
    def pg_start(self):
        # TODO: implement pg_hba.conf reconfig to allow cluster nodes to join the cluster automatically
        try:
            check_call(['systemctl', 'start', self._service()])
        except Exception:
            call(['systemctl', 'status', self._service()])
            raise
        self._wait_for_connections()
        if not self._is_active():
            raise Exception('Failed to start postgresql')

    @subscribe
    def pg_reload(self):
        check_call(['systemctl', 'reload', self._service()])

    @subscribe
    def pg_stop(self):
        check_call(['systemctl', 'stop', self._service()])

    @subscribe
    def pg_reset(self):
        self.pg_stop()
        try:
            check_call(['pg_dropcluster', '--stop', self._version, self._cluster_name])
        except:
            if os.path.exists(self._config_file()):
                check_call(['rm', '-rf', self._data_dir(), self._config_dir])

    @subscribe
    def pg_initdb(self):
        if os.path.exists(self._config_file()):
            self.pg_stop()
            check_call(['pg_dropcluster', '--stop', self._version, self._cluster_name])
        create_args = ['pg_createcluster', self._version, self._cluster_name]
        initdb_options = self.app.config['apt'].get('initdb_options', '').strip()
        if initdb_options:
            initdb_options = [i.strip() for i in initdb_options.split() if i.strip()]
            create_args.append('--')
            create_args.extend(initdb_options)
        check_call(['pg_createcluster', self._version, self._cluster_name, '--', '--data-checksums'])
        self._copy_config()
        self._set_config_values()
        if self._create_superuser:
            self.pg_start()
            check_call(['sudo', '-u', 'postgres', 'createuser', '-s', '-h', self._socket_dir(), '-p', self._port(), self._superuser_connect_as])
            self.pg_stop()

    @subscribe
    def pg_connect_info(self):
        info = dict(database='postgres', user=self._superuser_connect_as, host=self._socket_dir(), port=self._port())
        user = self._superuser_connect_as
        if user is not None:
            info['user'] = user
        return info

    @subscribe
    def get_conn_info(self):
        return dict(port=self._port())

    @subscribe
    def pg_replication_role(self):
        try:
            return os.path.exists(os.path.join(self._data_dir(), 'recovery.conf')) and 'replica' or 'master'
        except _NoCluster:
            return None

    @subscribe
    def start_monitoring(self):
        self.app.unhealthy(self._health_check_key, 'Waiting for first systemd check')
        loop = asyncio.get_event_loop()
        loop.call_soon(loop.create_task, self._monitor_systemd())

    def _is_active(self):
        return 0 == call(['systemctl', '--quiet', 'is-active', self._service()])

    async def _monitor_systemd(self):
        loop = asyncio.get_event_loop()
        while True:
            await sleep(1)
            if self._is_active():
                self.app.healthy(self._health_check_key)
            else:
                await sleep(2)
                if not self._is_active():
                    self.app.unhealthy(self._health_check_key, 'inactive according to systemd')

    def _trigger_file(self):
        return '/var/run/postgresql/{}-{}.master_trigger'.format(self._version, self._cluster_name)

    def _pg_is_in_recovery(self):
        conn = self._conn()
        try:
            cur = conn.cursor()
            cur.execute('SELECT pg_is_in_recovery();')
            return cur.fetchall()[0][0]
        finally:
            conn.close()

    @subscribe
    def pg_stop_replication(self):
        assert self.pg_replication_role() == 'replica'
        trigger_file = self._trigger_file()
        with open(trigger_file, 'w') as f:
            f.write('touched')
        time.sleep(0.2)
        count = 0
        while count < 60:
            try:
                if not self._pg_is_in_recovery():
                    break
            except psycopg2.OperationalError as e:
                pass
            logging.info('waiting for postgresql to come out of recovery')
            count += 1
            time.sleep(1)
        else:
            # hmm, what to do here?
            raise Exception('postgresql did not come out of recovery mode')
        if self._set_config_values('master.'):
            self.pg_reload()

    @subscribe
    def pg_setup_replication(self, primary_conninfo=None):
        trigger_file = self._trigger_file()
        if os.path.exists(trigger_file):
            os.remove(trigger_file)
        config = """standby_mode = 'on'
trigger_file = '{trigger_file}'
recovery_target_timeline = 'latest'
"""
        config = config.format(trigger_file=trigger_file)
        restore_command = self.app.config['apt'].get('restore_command', None)
        if primary_conninfo:
            parts = []
            for k, v in primary_conninfo.items():
                parts.append('{}={}'.format(k, v))
            config += "\nprimary_conninfo = '{}'".format(' '.join(parts))
        if restore_command:
            config += "\nrestore_command = '{}'".format(restore_command)
        with open(os.path.join(self._data_dir(), 'recovery.conf'), 'w') as f:
            f.write(config)
        self._set_config_values('replica.')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Interface to the Crassus Lambda function. This module notifies Crassus
about updates to a CFN stack so Crassus will trigger the update process.
"""

import json
import logging
from datetime import datetime, timedelta
from dateutil import tz, parser as datetime_parser
from time import sleep

import boto3

logger = logging.getLogger('gaius')

FINAL_STATES = [
    'CREATE_COMPLETE',
    'UPDATE_COMPLETE',
]

ERROR_STATES = [
    'CREATE_FAILED',
    'ROLLBACK_FAILED',
    'DELETE_FAILED',
    'ROLLBACK_COMPLETE',
    'UPDATE_ROLLBACK_COMPLETE',
    'UPDATE_ROLLBACK_FAILED',
    'DELETE_COMPLETE'
]

# Empty credentials per default, so the environment ones will be used
_credentials = {
    'AccessKeyId': None,
    'SecretAccessKey': None,
    'SessionToken': None
}

def credentials_set(credentials):
    """
    Set the module credentials for when a user wants to use other ones
    than given in the environment.
    """
    _credentials.update({
        'AccessKeyId': credentials.get('AccessKeyId'),
        'SecretAccessKey': credentials.get('SecretAccessKey'),
        'SessionToken': credentials.get('SessionToken')
    })

def credentials_reset():
    """
    Reset (empty) the module credentials so the commands will use the
    ones specified by the environment.
    """
    _credentials.update({
        'AccessKeyId': None,
        'SecretAccessKey': None,
        'SessionToken': None
    })

def parse_parameters(parameters):
    """ Parse input parameters from the command line """
    parameter_list = [x for x in parameters.split(',')]
    return dict([y.split('=') for y in parameter_list])

def generate_message(stack_name, parameters, region, version=1):
    """ Generate the update notification message """
    message = {}
    message['version'] = version
    message['stackName'] = stack_name
    message['region'] = region
    message['parameters'] = parse_parameters(parameters)
    return message

def notify(stack_name, parameters, topic_arn, region):
    """ Sends an update notification to Crassus """
    message = generate_message(stack_name, parameters, region)
    sns_resource = boto3.resource(
        'sns', region_name=region,
        aws_access_key_id=_credentials['AccessKeyId'],
        aws_secret_access_key=_credentials['SecretAccessKey'],
        aws_session_token=_credentials['SessionToken'])
    sns_topic = sns_resource.Topic(topic_arn)
    result = sns_topic.publish(Message=json.dumps(message))
    logger.debug(result)
    return result

def is_related_message(message_dict, stack_name):
    """Checks if StackName belongs to client-session or is  missing"""
    if message_dict.get('stackName') == stack_name:
        return True
    elif message_dict.get('stackName') is None:
        return True
    return False

def cleanup(back_channel_url, duration,  stack_name, region):
    """Cleans up old messages on the deployment pipeline"""
    sqs_resource = boto3.resource(
        'sqs', region_name=region,
        aws_access_key_id=_credentials['AccessKeyId'],
        aws_secret_access_key=_credentials['SecretAccessKey'],
        aws_session_token=_credentials['SessionToken'])
    queue = sqs_resource.Queue(url=back_channel_url)
    end_time = datetime.now() + timedelta(seconds=duration)

    while datetime.now() <= end_time:
        messages = queue.receive_messages(MaxNumberOfMessages=10)
        messages = filter_stack_related_messages(messages, stack_name)
        if messages:
            for message in messages:
                cleanup_old_messages(message, stack_name)
                sleep(1)

def filter_stack_related_messages(messages, stack_name):
    return filter(
        lambda msg: json.loads(msg.body).get('stackName') == stack_name,
        messages)

def log_delete_message(message_dict):
    message_status = message_dict.get('status')
    message_payload = message_dict.get('message')
    message_rtype = message_dict.get('resourceType')
    message_stack_name = message_dict['stackName']
    logger.info('%s: %s: %s: %s', message_stack_name,
                message_status, message_rtype, message_payload)

def cleanup_old_messages(message, stack_name):
    now = datetime.now(tz=tz.tzutc())
    message_dict = json.loads(message.body)
    message_stack_name = message_dict['stackName']
    message_timestamp = message_dict['timestamp']
    message_datetime = datetime_parser.parse(message_timestamp)
    if (message_stack_name == stack_name and
            message_datetime < now):
        log_delete_message(message_dict)
        message.delete()
        return True

def receive(back_channel_url, timeout,  stack_name, region,
            poll_interval=2):
    """Reads out the back-channel on the deployment pipeline"""
    timeout_orig = timeout
    sqs_resource = boto3.resource(
        'sqs', region_name=region,
        aws_access_key_id=_credentials['AccessKeyId'],
        aws_secret_access_key=_credentials['SecretAccessKey'],
        aws_session_token=_credentials['SessionToken'])
    queue = sqs_resource.Queue(url=back_channel_url)
    while timeout > 0:
        messages = queue.receive_messages(MaxNumberOfMessages=1)
        for message in messages:
            if process_message(message, stack_name):
                cleanup(back_channel_url, timeout, stack_name, region)
                logger.info('Final CFN message received')
                return
        timeout -= poll_interval
        sleep(poll_interval)
    # raise exception if we reach this point as presumably no final stage
    # is reached within the timeout
    raise DeploymentErrorException(
        'No final CFN message was received after {0} seconds'.format(
            timeout_orig))

def process_message(message, stack_name):
    message_dict = json.loads(message.body)
    message_status = message_dict.get('status')
    message_payload = message_dict.get('message')
    message_rtype = message_dict.get('resourceType')

    logger.debug(message_dict)
    if not is_related_message(message_dict, stack_name):
        message.change_visibility(VisibilityTimeout=0)
    else:
        log_delete_message(message_dict)
        message.delete()
        if message_status == 'failure':
            raise DeploymentErrorException(
                'Crassus failed with "{0}"'.format(message_payload))
        elif (message_rtype ==
              'AWS::CloudFormation::Stack' and message_status in ERROR_STATES):
            raise DeploymentErrorException(
                'Crassus failed with "{0}"'.format(message_payload))
        elif (message_rtype ==
              'AWS::CloudFormation::Stack' and message_status in FINAL_STATES):
            return True

class DeploymentErrorException(Exception):

    def __init__(self, message):
        self.message = message

    def __str__(self):
        return self.message
<commit_msg>Don't know why we do this cleanup after finding the final message. Maybe it's a good idea, maybe it's not. IMHO it won't hurt if we do it for 10 seconds.<commit_after># -*- coding: utf-8 -*-
"""
Interface to the Crassus Lambda function. This module notifies Crassus
about updates to a CFN stack so Crassus will trigger the update process.
"""

import json
import logging
from datetime import datetime, timedelta
from dateutil import tz, parser as datetime_parser
from time import sleep

import boto3

logger = logging.getLogger('gaius')

FINAL_STATES = [
    'CREATE_COMPLETE',
    'UPDATE_COMPLETE',
]

ERROR_STATES = [
    'CREATE_FAILED',
    'ROLLBACK_FAILED',
    'DELETE_FAILED',
    'ROLLBACK_COMPLETE',
    'UPDATE_ROLLBACK_COMPLETE',
    'UPDATE_ROLLBACK_FAILED',
    'DELETE_COMPLETE'
]

# Empty credentials per default, so the environment ones will be used
_credentials = {
    'AccessKeyId': None,
    'SecretAccessKey': None,
    'SessionToken': None
}

def credentials_set(credentials):
    """
    Set the module credentials for when a user wants to use other ones
    than given in the environment.
    """
    _credentials.update({
        'AccessKeyId': credentials.get('AccessKeyId'),
        'SecretAccessKey': credentials.get('SecretAccessKey'),
        'SessionToken': credentials.get('SessionToken')
    })

def credentials_reset():
    """
    Reset (empty) the module credentials so the commands will use the
    ones specified by the environment.
    """
    _credentials.update({
        'AccessKeyId': None,
        'SecretAccessKey': None,
        'SessionToken': None
    })

def parse_parameters(parameters):
    """ Parse input parameters from the command line """
    parameter_list = [x for x in parameters.split(',')]
    return dict([y.split('=') for y in parameter_list])

def generate_message(stack_name, parameters, region, version=1):
    """ Generate the update notification message """
    message = {}
    message['version'] = version
    message['stackName'] = stack_name
    message['region'] = region
    message['parameters'] = parse_parameters(parameters)
    return message

def notify(stack_name, parameters, topic_arn, region):
    """ Sends an update notification to Crassus """
    message = generate_message(stack_name, parameters, region)
    sns_resource = boto3.resource(
        'sns', region_name=region,
        aws_access_key_id=_credentials['AccessKeyId'],
        aws_secret_access_key=_credentials['SecretAccessKey'],
        aws_session_token=_credentials['SessionToken'])
    sns_topic = sns_resource.Topic(topic_arn)
    result = sns_topic.publish(Message=json.dumps(message))
    logger.debug(result)
    return result

def is_related_message(message_dict, stack_name):
    """Checks if StackName belongs to client-session or is  missing"""
    if message_dict.get('stackName') == stack_name:
        return True
    elif message_dict.get('stackName') is None:
        return True
    return False

def cleanup(back_channel_url, duration,  stack_name, region):
    """Cleans up old messages on the deployment pipeline"""
    sqs_resource = boto3.resource(
        'sqs', region_name=region,
        aws_access_key_id=_credentials['AccessKeyId'],
        aws_secret_access_key=_credentials['SecretAccessKey'],
        aws_session_token=_credentials['SessionToken'])
    queue = sqs_resource.Queue(url=back_channel_url)
    end_time = datetime.now() + timedelta(seconds=duration)

    while datetime.now() <= end_time:
        messages = queue.receive_messages(MaxNumberOfMessages=10)
        messages = filter_stack_related_messages(messages, stack_name)
        if messages:
            for message in messages:
                cleanup_old_messages(message, stack_name)
                sleep(1)

def filter_stack_related_messages(messages, stack_name):
    return filter(
        lambda msg: json.loads(msg.body).get('stackName') == stack_name,
        messages)

def log_delete_message(message_dict):
    message_status = message_dict.get('status')
    message_payload = message_dict.get('message')
    message_rtype = message_dict.get('resourceType')
    message_stack_name = message_dict['stackName']
    logger.info('%s: %s: %s: %s', message_stack_name,
                message_status, message_rtype, message_payload)

def cleanup_old_messages(message, stack_name):
    now = datetime.now(tz=tz.tzutc())
    message_dict = json.loads(message.body)
    message_stack_name = message_dict['stackName']
    message_timestamp = message_dict['timestamp']
    message_datetime = datetime_parser.parse(message_timestamp)
    if (message_stack_name == stack_name and
            message_datetime < now):
        log_delete_message(message_dict)
        message.delete()
        return True

def receive(back_channel_url, timeout,  stack_name, region,
            poll_interval=2):
    """Reads out the back-channel on the deployment pipeline"""
    timeout_orig = timeout
    sqs_resource = boto3.resource(
        'sqs', region_name=region,
        aws_access_key_id=_credentials['AccessKeyId'],
        aws_secret_access_key=_credentials['SecretAccessKey'],
        aws_session_token=_credentials['SessionToken'])
    queue = sqs_resource.Queue(url=back_channel_url)
    while timeout > 0:
        messages = queue.receive_messages(MaxNumberOfMessages=1)
        for message in messages:
            if process_message(message, stack_name):
                cleanup(back_channel_url, 10, stack_name, region)
                logger.info('Final CFN message received')
                return
        timeout -= poll_interval
        sleep(poll_interval)
    # raise exception if we reach this point as presumably no final stage
    # is reached within the timeout
    raise DeploymentErrorException(
        'No final CFN message was received after {0} seconds'.format(
            timeout_orig))

def process_message(message, stack_name):
    message_dict = json.loads(message.body)
    message_status = message_dict.get('status')
    message_payload = message_dict.get('message')
    message_rtype = message_dict.get('resourceType')

    logger.debug(message_dict)
    if not is_related_message(message_dict, stack_name):
        message.change_visibility(VisibilityTimeout=0)
    else:
        log_delete_message(message_dict)
        message.delete()
        if message_status == 'failure':
            raise DeploymentErrorException(
                'Crassus failed with "{0}"'.format(message_payload))
        elif (message_rtype ==
              'AWS::CloudFormation::Stack' and message_status in ERROR_STATES):
            raise DeploymentErrorException(
                'Crassus failed with "{0}"'.format(message_payload))
        elif (message_rtype ==
              'AWS::CloudFormation::Stack' and message_status in FINAL_STATES):
            return True

class DeploymentErrorException(Exception):

    def __init__(self, message):
        self.message = message

    def __str__(self):
        return self.message
<|endoftext|>


<commit_before>from neuroimaging.utils.testutils import make_doctest_suite
test_suite = make_doctest_suite('neuroimaging.algorithms.statistics.classifcation')

if __name__ == '__main__':
    NumpyTest.run()
<commit_msg>BUG: Fix typo in test_classification.<commit_after>from neuroimaging.utils.testutils import make_doctest_suite
test_suite = make_doctest_suite('neuroimaging.algorithms.statistics.classification')

if __name__ == '__main__':
    NumpyTest.run()
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2021 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Small models to be finetuned on embeddings."""

from sklearn import discriminant_analysis
from sklearn import ensemble
from sklearn import linear_model

LogisticRegression = linear_model.LogisticRegression
LinearDiscriminantAnalysis = discriminant_analysis.LinearDiscriminantAnalysis
RandomForestClassifier = ensemble.RandomForestClassifier

# pylint:disable=line-too-long,unused-variable, g-no-space-after-comment
def get_sklearn_models():
  return {
      'LogisticRegression': lambda: LogisticRegression(C=1e5, solver='lbfgs', multi_class='multinomial'),
      'LogisticRegression_balanced': lambda: LogisticRegression(C=1e5, solver='lbfgs', multi_class='multinomial', class_weight='balanced'),
      'LDA_LSQR_AUTO': lambda: LinearDiscriminantAnalysis(solver='lsqr', shrinkage='auto'),
      'Forest_balanced_100': lambda: RandomForestClassifier(class_weight='balanced', n_estimators=100),
      'Forest_unbalanced_100': lambda: RandomForestClassifier(class_weight=None, n_estimators=100),
  }
# pylint:enable=line-too-long,unused-variable, g-no-space-after-comment
<commit_msg>Remove boosted trees. They're not the best on any NOSS task.<commit_after># coding=utf-8
# Copyright 2021 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Small models to be finetuned on embeddings."""

from sklearn import discriminant_analysis
from sklearn import ensemble
from sklearn import linear_model

LogisticRegression = linear_model.LogisticRegression
LinearDiscriminantAnalysis = discriminant_analysis.LinearDiscriminantAnalysis
RandomForestClassifier = ensemble.RandomForestClassifier

# pylint:disable=line-too-long,unused-variable, g-no-space-after-comment
def get_sklearn_models():
  return {
      'LogisticRegression':
          lambda: LogisticRegression(
              C=1e5, solver='lbfgs', multi_class='multinomial'),
      'LogisticRegression_balanced':
          lambda: LogisticRegression(
              C=1e5,
              solver='lbfgs',
              multi_class='multinomial',
              class_weight='balanced'),
      'LDA_LSQR_AUTO':
          lambda: LinearDiscriminantAnalysis(solver='lsqr', shrinkage='auto'),
  }
# pylint:enable=line-too-long,unused-variable, g-no-space-after-comment
<|endoftext|>