<commit_before>#
# Cassandra Cluster Management lib
#

import os
import re
import shutil
import socket
import stat
import subprocess
import sys
from six import print_
import time
import yaml

CASSANDRA_BIN_DIR= "bin"
CASSANDRA_CONF_DIR= "conf"

CASSANDRA_CONF = "cassandra.yaml"
LOG4J_CONF = "log4j-server.properties"
LOG4J_TOOL_CONF = "log4j-tools.properties"
LOGBACK_CONF = "logback.xml"
CASSANDRA_ENV = "cassandra-env.sh"
CASSANDRA_WIN_ENV = "cassandra-env.ps1"
CASSANDRA_SH = "cassandra.in.sh"

CONFIG_FILE = "config"

class CCMError(Exception):
    pass

class LoadError(CCMError):
    pass

class ArgumentError(CCMError):
    pass

class UnavailableSocketError(CCMError):
    pass

def get_default_path():
    default_path = os.path.join(get_user_home(), '.ccm')
    if not os.path.exists(default_path):
        os.mkdir(default_path)
    return default_path

def get_user_home():
    if is_win():
        if sys.platform == "cygwin":
            # Need the fully qualified directory
            output = subprocess.Popen(["cygpath", "-m", os.path.expanduser('~')], stdout = subprocess.PIPE, stderr = subprocess.STDOUT).communicate()[0].rstrip()
            return output
        else:
            return os.environ['USERPROFILE']
    else:
        return os.path.expanduser('~')

def get_config():
    config_path = os.path.join(get_default_path(), CONFIG_FILE)
    if not os.path.exists(config_path):
        return {}

    with open(config_path, 'r') as f:
        return yaml.load(f)

def now_ms():
    return int(round(time.time() * 1000))

def parse_interface(itf, default_port):
    i = itf.split(':')
    if len(i) == 1:
        return (i[0].strip(), default_port)
    elif len(i) == 2:
        return (i[0].strip(), int(i[1].strip()))
    else:
        raise ValueError("Invalid interface definition: " + itf)

def current_cluster_name(path):
    try:
        with open(os.path.join(path, 'CURRENT'), 'r') as f:
            return f.readline().strip()
    except IOError:
        return None

def switch_cluster(path, new_name):
    with open(os.path.join(path, 'CURRENT'), 'w') as f:
        f.write(new_name + '\n')

def replace_in_file(file, regexp, replace):
    replaces_in_file(file, [(regexp, replace)])

def replaces_in_file(file, replacement_list):
    rs = [ (re.compile(regexp), repl) for (regexp, repl) in replacement_list]
    file_tmp = file + ".tmp"
    with open(file, 'r') as f:
        with open(file_tmp, 'w') as f_tmp:
            for line in f:
                for r, replace in rs:
                    match = r.search(line)
                    if match:
                        line = replace + "\n"
                f_tmp.write(line)
    shutil.move(file_tmp, file)

def replace_or_add_into_file_tail(file, regexp, replace):
    replaces_or_add_into_file_tail(file, [(regexp, replace)])

def replaces_or_add_into_file_tail(file, replacement_list):
    rs = [ (re.compile(regexp), repl) for (regexp, repl) in replacement_list]
    is_line_found = False
    file_tmp = file + ".tmp"
    with open(file, 'r') as f:
        with open(file_tmp, 'w') as f_tmp:
            for line in f:
                for r, replace in rs:
                    match = r.search(line)
                    if match:
                        line = replace + "\n"
                        is_line_found = True
                f_tmp.write(line)
            # In case, entry is not found, and need to be added
            if is_line_found == False:
                f_tmp.write('\n'+ replace + "\n")

    shutil.move(file_tmp, file)

def make_cassandra_env(cassandra_dir, node_path):
    if is_win() and get_version_from_build(node_path=node_path) >= '2.1':
        sh_file = os.path.join(CASSANDRA_CONF_DIR, CASSANDRA_WIN_ENV)
    else:
        sh_file = os.path.join(CASSANDRA_BIN_DIR, CASSANDRA_SH)
    orig = os.path.join(cassandra_dir, sh_file)
    dst = os.path.join(node_path, sh_file)
    shutil.copy(orig, dst)
    replacements = ""
    if is_win() and get_version_from_build(node_path=node_path) >= '2.1':
        replacements = [
            ('env:CASSANDRA_HOME =', '        $env:CASSANDRA_HOME="%s"' % cassandra_dir),
            ('env:CASSANDRA_CONF =', '    $env:CASSANDRA_CONF="%s"' % os.path.join(node_path, 'conf'))
        ]
    else:
        replacements = [
            ('CASSANDRA_HOME=', '\tCASSANDRA_HOME=%s' % cassandra_dir),
            ('CASSANDRA_CONF=', '\tCASSANDRA_CONF=%s' % os.path.join(node_path, 'conf'))
        ]
    replaces_in_file(dst, replacements)

    # If a cluster-wide cassandra.in.sh file exists in the parent
    # directory, append it to the node specific one:
    cluster_sh_file = os.path.join(node_path, os.path.pardir, 'cassandra.in.sh')
    if os.path.exists(cluster_sh_file):
        append = open(cluster_sh_file).read()
        with open(dst, 'a') as f:
            f.write('\n\n### Start Cluster wide config ###\n')
            f.write(append)
            f.write('\n### End Cluster wide config ###\n\n')

    env = os.environ.copy()
    env['CASSANDRA_INCLUDE'] = os.path.join(dst)
    env['MAX_HEAP_SIZE'] = os.environ.get('CCM_MAX_HEAP_SIZE', '500M')
    env['HEAP_NEWSIZE'] = os.environ.get('CCM_HEAP_NEWSIZE', '50M')

    return env

def check_win_requirements():
    if is_win():
        # Make sure ant.bat is in the path and executable before continuing
        try:
            process = subprocess.Popen('ant.bat', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        except Exception as e:
            sys.exit("ERROR!  Could not find or execute ant.bat.  Please fix this before attempting to run ccm on Windows.")

def is_win():
    return True if sys.platform == "cygwin" or sys.platform == "win32" else False

def join_bin(root, dir, executable):
    return os.path.join(root, dir, platform_binary(executable))

def platform_binary(input):
    return input + ".bat" if is_win() else input

def platform_pager():
    return "more" if sys.platform == "win32" else "less"

def add_exec_permission(path, executable):
    # 1) os.chmod on Windows can't add executable permissions
    # 2) chmod from other folders doesn't work in cygwin, so we have to navigate the shell
    # to the folder with the executable with it and then chmod it from there
    if sys.platform == "cygwin":
        cmd = "cd " + path + "; chmod u+x " + executable
        os.system(cmd)

def parse_path(executable):
    sep = os.sep
    if sys.platform == "win32":
        sep = "\\\\"
    tokens = re.split(sep, executable)
    del tokens[-1]
    return os.sep.join(tokens)

def parse_bin(executable):
    tokens = re.split(os.sep, executable)
    return tokens[-1]

def get_stress_bin(cassandra_dir):
    candidates = [
        os.path.join(cassandra_dir, 'contrib', 'stress', 'bin', 'stress'),
        os.path.join(cassandra_dir, 'tools', 'stress', 'bin', 'stress'),
        os.path.join(cassandra_dir, 'tools', 'bin', 'stress'),
        os.path.join(cassandra_dir, 'tools', 'bin', 'cassandra-stress')
    ]
    candidates = [platform_binary(s) for s in candidates]

    for candidate in candidates:
        if os.path.exists(candidate):
            stress = candidate
            break
    else:
        raise Exception("Cannot find stress binary (maybe it isn't compiled)")

    # make sure it's executable -> win32 doesn't care
    if sys.platform == "cygwin":
        # Yes, we're unwinding the path join from above.
        path = parse_path(stress)
        short_bin = parse_bin(stress)
        add_exec_permission(path, short_bin)
    elif not os.access(stress, os.X_OK):
        try:
            # try to add user execute permissions
            # os.chmod doesn't work on Windows and isn't necessary unless in cygwin...
            if sys.platform == "cygwin":
                add_exec_permission(path, stress)
            else:
                os.chmod(stress, os.stat(stress).st_mode | stat.S_IXUSR)
        except:
            raise Exception("stress binary is not executable: %s" % (stress,))

    return stress

def validate_cassandra_dir(cassandra_dir):
    if cassandra_dir is None:
        raise ArgumentError('Undefined cassandra directory')

    # Windows requires absolute pathing on cassandra dir - abort if specified cygwin style
    if is_win():
        if ':' not in cassandra_dir:
            raise ArgumentError('%s does not appear to be a cassandra source directory.  Please use absolute pathing (e.g. C:/cassandra.' % cassandra_dir)

    bin_dir = os.path.join(cassandra_dir, CASSANDRA_BIN_DIR)
    conf_dir = os.path.join(cassandra_dir, CASSANDRA_CONF_DIR)
    cnd = os.path.exists(bin_dir)
    cnd = cnd and os.path.exists(conf_dir)
    cnd = cnd and os.path.exists(os.path.join(conf_dir, CASSANDRA_CONF))
    if not cnd:
        raise ArgumentError('%s does not appear to be a cassandra source directory' % cassandra_dir)

def check_socket_available(itf):
    s = socket.socket()
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    try:
        s.bind(itf)
        s.close()
    except socket.error as msg:
        s.close()
        addr, port = itf
        raise UnavailableSocketError("Inet address %s:%s is not available: %s" % (addr, port, msg))

def parse_settings(args):
    settings = {}
    for s in args:
        splitted = s.split(':')
        if len(splitted) != 2:
            raise ArgumentError("A new setting should be of the form 'key: value', got" + s)
        val = splitted[1].strip()
        # ok, that's not super beautiful
        if val.lower() == "true":
            val = True
        if val.lower() == "false":
            val = True
        try:
            val = int(val)
        except ValueError:
            pass
        settings[splitted[0].strip()] = val
    return settings

#
# Copy file from source to destination with reasonable error handling
#
def copy_file(src_file, dst_file):
    try:
        shutil.copy2(src_file, dst_file)
    except (IOError, shutil.Error) as e:
        print_(str(e), file=sys.stderr)
        exit(1)

def get_version_from_build(cassandra_dir=None, node_path=None):
    if cassandra_dir is None and node_path is not None:
        cassandra_dir = get_cassandra_dir_from_cluster_conf(node_path)
    if cassandra_dir is not None:
        build = os.path.join(cassandra_dir, 'build.xml')
        with open(build) as f:
            for line in f:
                match = re.search('name="base\.version" value="([0-9.]+)[^"]*"', line)
                if match:
                    return match.group(1)
    raise CCMError("Cannot find version")

def get_cassandra_dir_from_cluster_conf(node_path):
    file = os.path.join(os.path.dirname(node_path), "cluster.conf")
    with open(file) as f:
        for line in f:
            match = re.search('cassandra_dir: (.*?)$', line)
            if match:
                return match.group(1)
    return None
<commit_msg>add arch check to win req function<commit_after>#
# Cassandra Cluster Management lib
#

import os
import platform
import re
import shutil
import socket
import stat
import subprocess
import sys
from six import print_
import time
import yaml

CASSANDRA_BIN_DIR= "bin"
CASSANDRA_CONF_DIR= "conf"

CASSANDRA_CONF = "cassandra.yaml"
LOG4J_CONF = "log4j-server.properties"
LOG4J_TOOL_CONF = "log4j-tools.properties"
LOGBACK_CONF = "logback.xml"
CASSANDRA_ENV = "cassandra-env.sh"
CASSANDRA_WIN_ENV = "cassandra-env.ps1"
CASSANDRA_SH = "cassandra.in.sh"

CONFIG_FILE = "config"

class CCMError(Exception):
    pass

class LoadError(CCMError):
    pass

class ArgumentError(CCMError):
    pass

class UnavailableSocketError(CCMError):
    pass

def get_default_path():
    default_path = os.path.join(get_user_home(), '.ccm')
    if not os.path.exists(default_path):
        os.mkdir(default_path)
    return default_path

def get_user_home():
    if is_win():
        if sys.platform == "cygwin":
            # Need the fully qualified directory
            output = subprocess.Popen(["cygpath", "-m", os.path.expanduser('~')], stdout = subprocess.PIPE, stderr = subprocess.STDOUT).communicate()[0].rstrip()
            return output
        else:
            return os.environ['USERPROFILE']
    else:
        return os.path.expanduser('~')

def get_config():
    config_path = os.path.join(get_default_path(), CONFIG_FILE)
    if not os.path.exists(config_path):
        return {}

    with open(config_path, 'r') as f:
        return yaml.load(f)

def now_ms():
    return int(round(time.time() * 1000))

def parse_interface(itf, default_port):
    i = itf.split(':')
    if len(i) == 1:
        return (i[0].strip(), default_port)
    elif len(i) == 2:
        return (i[0].strip(), int(i[1].strip()))
    else:
        raise ValueError("Invalid interface definition: " + itf)

def current_cluster_name(path):
    try:
        with open(os.path.join(path, 'CURRENT'), 'r') as f:
            return f.readline().strip()
    except IOError:
        return None

def switch_cluster(path, new_name):
    with open(os.path.join(path, 'CURRENT'), 'w') as f:
        f.write(new_name + '\n')

def replace_in_file(file, regexp, replace):
    replaces_in_file(file, [(regexp, replace)])

def replaces_in_file(file, replacement_list):
    rs = [ (re.compile(regexp), repl) for (regexp, repl) in replacement_list]
    file_tmp = file + ".tmp"
    with open(file, 'r') as f:
        with open(file_tmp, 'w') as f_tmp:
            for line in f:
                for r, replace in rs:
                    match = r.search(line)
                    if match:
                        line = replace + "\n"
                f_tmp.write(line)
    shutil.move(file_tmp, file)

def replace_or_add_into_file_tail(file, regexp, replace):
    replaces_or_add_into_file_tail(file, [(regexp, replace)])

def replaces_or_add_into_file_tail(file, replacement_list):
    rs = [ (re.compile(regexp), repl) for (regexp, repl) in replacement_list]
    is_line_found = False
    file_tmp = file + ".tmp"
    with open(file, 'r') as f:
        with open(file_tmp, 'w') as f_tmp:
            for line in f:
                for r, replace in rs:
                    match = r.search(line)
                    if match:
                        line = replace + "\n"
                        is_line_found = True
                f_tmp.write(line)
            # In case, entry is not found, and need to be added
            if is_line_found == False:
                f_tmp.write('\n'+ replace + "\n")

    shutil.move(file_tmp, file)

def make_cassandra_env(cassandra_dir, node_path):
    if is_win() and get_version_from_build(node_path=node_path) >= '2.1':
        sh_file = os.path.join(CASSANDRA_CONF_DIR, CASSANDRA_WIN_ENV)
    else:
        sh_file = os.path.join(CASSANDRA_BIN_DIR, CASSANDRA_SH)
    orig = os.path.join(cassandra_dir, sh_file)
    dst = os.path.join(node_path, sh_file)
    shutil.copy(orig, dst)
    replacements = ""
    if is_win() and get_version_from_build(node_path=node_path) >= '2.1':
        replacements = [
            ('env:CASSANDRA_HOME =', '        $env:CASSANDRA_HOME="%s"' % cassandra_dir),
            ('env:CASSANDRA_CONF =', '    $env:CASSANDRA_CONF="%s"' % os.path.join(node_path, 'conf'))
        ]
    else:
        replacements = [
            ('CASSANDRA_HOME=', '\tCASSANDRA_HOME=%s' % cassandra_dir),
            ('CASSANDRA_CONF=', '\tCASSANDRA_CONF=%s' % os.path.join(node_path, 'conf'))
        ]
    replaces_in_file(dst, replacements)

    # If a cluster-wide cassandra.in.sh file exists in the parent
    # directory, append it to the node specific one:
    cluster_sh_file = os.path.join(node_path, os.path.pardir, 'cassandra.in.sh')
    if os.path.exists(cluster_sh_file):
        append = open(cluster_sh_file).read()
        with open(dst, 'a') as f:
            f.write('\n\n### Start Cluster wide config ###\n')
            f.write(append)
            f.write('\n### End Cluster wide config ###\n\n')

    env = os.environ.copy()
    env['CASSANDRA_INCLUDE'] = os.path.join(dst)
    env['MAX_HEAP_SIZE'] = os.environ.get('CCM_MAX_HEAP_SIZE', '500M')
    env['HEAP_NEWSIZE'] = os.environ.get('CCM_HEAP_NEWSIZE', '50M')

    return env

def check_win_requirements():
    if is_win():
        # Make sure ant.bat is in the path and executable before continuing
        try:
            process = subprocess.Popen('ant.bat', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        except Exception as e:
            sys.exit("ERROR!  Could not find or execute ant.bat.  Please fix this before attempting to run ccm on Windows.")

        # Confirm matching architectures
        # 32-bit python distributions will launch 32-bit cmd environments, losing PowerShell execution privileges on a 64-bit system
        if sys.maxsize <= 2**32 and platform.machine().endswith('64'):
            sys.exit("ERROR!  64-bit os and 32-bit python distribution found.  ccm requires matching architectures.")

def is_win():
    return True if sys.platform == "cygwin" or sys.platform == "win32" else False

def join_bin(root, dir, executable):
    return os.path.join(root, dir, platform_binary(executable))

def platform_binary(input):
    return input + ".bat" if is_win() else input

def platform_pager():
    return "more" if sys.platform == "win32" else "less"

def add_exec_permission(path, executable):
    # 1) os.chmod on Windows can't add executable permissions
    # 2) chmod from other folders doesn't work in cygwin, so we have to navigate the shell
    # to the folder with the executable with it and then chmod it from there
    if sys.platform == "cygwin":
        cmd = "cd " + path + "; chmod u+x " + executable
        os.system(cmd)

def parse_path(executable):
    sep = os.sep
    if sys.platform == "win32":
        sep = "\\\\"
    tokens = re.split(sep, executable)
    del tokens[-1]
    return os.sep.join(tokens)

def parse_bin(executable):
    tokens = re.split(os.sep, executable)
    return tokens[-1]

def get_stress_bin(cassandra_dir):
    candidates = [
        os.path.join(cassandra_dir, 'contrib', 'stress', 'bin', 'stress'),
        os.path.join(cassandra_dir, 'tools', 'stress', 'bin', 'stress'),
        os.path.join(cassandra_dir, 'tools', 'bin', 'stress'),
        os.path.join(cassandra_dir, 'tools', 'bin', 'cassandra-stress')
    ]
    candidates = [platform_binary(s) for s in candidates]

    for candidate in candidates:
        if os.path.exists(candidate):
            stress = candidate
            break
    else:
        raise Exception("Cannot find stress binary (maybe it isn't compiled)")

    # make sure it's executable -> win32 doesn't care
    if sys.platform == "cygwin":
        # Yes, we're unwinding the path join from above.
        path = parse_path(stress)
        short_bin = parse_bin(stress)
        add_exec_permission(path, short_bin)
    elif not os.access(stress, os.X_OK):
        try:
            # try to add user execute permissions
            # os.chmod doesn't work on Windows and isn't necessary unless in cygwin...
            if sys.platform == "cygwin":
                add_exec_permission(path, stress)
            else:
                os.chmod(stress, os.stat(stress).st_mode | stat.S_IXUSR)
        except:
            raise Exception("stress binary is not executable: %s" % (stress,))

    return stress

def validate_cassandra_dir(cassandra_dir):
    if cassandra_dir is None:
        raise ArgumentError('Undefined cassandra directory')

    # Windows requires absolute pathing on cassandra dir - abort if specified cygwin style
    if is_win():
        if ':' not in cassandra_dir:
            raise ArgumentError('%s does not appear to be a cassandra source directory.  Please use absolute pathing (e.g. C:/cassandra.' % cassandra_dir)

    bin_dir = os.path.join(cassandra_dir, CASSANDRA_BIN_DIR)
    conf_dir = os.path.join(cassandra_dir, CASSANDRA_CONF_DIR)
    cnd = os.path.exists(bin_dir)
    cnd = cnd and os.path.exists(conf_dir)
    cnd = cnd and os.path.exists(os.path.join(conf_dir, CASSANDRA_CONF))
    if not cnd:
        raise ArgumentError('%s does not appear to be a cassandra source directory' % cassandra_dir)

def check_socket_available(itf):
    s = socket.socket()
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    try:
        s.bind(itf)
        s.close()
    except socket.error as msg:
        s.close()
        addr, port = itf
        raise UnavailableSocketError("Inet address %s:%s is not available: %s" % (addr, port, msg))

def parse_settings(args):
    settings = {}
    for s in args:
        splitted = s.split(':')
        if len(splitted) != 2:
            raise ArgumentError("A new setting should be of the form 'key: value', got" + s)
        val = splitted[1].strip()
        # ok, that's not super beautiful
        if val.lower() == "true":
            val = True
        if val.lower() == "false":
            val = True
        try:
            val = int(val)
        except ValueError:
            pass
        settings[splitted[0].strip()] = val
    return settings

#
# Copy file from source to destination with reasonable error handling
#
def copy_file(src_file, dst_file):
    try:
        shutil.copy2(src_file, dst_file)
    except (IOError, shutil.Error) as e:
        print_(str(e), file=sys.stderr)
        exit(1)

def get_version_from_build(cassandra_dir=None, node_path=None):
    if cassandra_dir is None and node_path is not None:
        cassandra_dir = get_cassandra_dir_from_cluster_conf(node_path)
    if cassandra_dir is not None:
        build = os.path.join(cassandra_dir, 'build.xml')
        with open(build) as f:
            for line in f:
                match = re.search('name="base\.version" value="([0-9.]+)[^"]*"', line)
                if match:
                    return match.group(1)
    raise CCMError("Cannot find version")

def get_cassandra_dir_from_cluster_conf(node_path):
    file = os.path.join(os.path.dirname(node_path), "cluster.conf")
    with open(file) as f:
        for line in f:
            match = re.search('cassandra_dir: (.*?)$', line)
            if match:
                return match.group(1)
    return None
<|endoftext|>


<commit_before>"""

  This module contains functions for building and loading NMODL mechanisms

  Author: Thomas G. Close (tclose@oist.jp)
  Copyright: 2012-2014 Thomas G. Close.
  License: This file is part of the "NineLine" package, which is released under
           the MIT Licence, see LICENSE for details.
"""
from __future__ import absolute_import
import os.path
import shutil
import time
import platform
import subprocess as sp

from .__init__ import (path_to_exec, get_build_paths,
                       load_component_translations)

BUILD_ARCHS = [platform.machine(), 'i686', 'x86_64', 'powerpc', 'umac']
_SIMULATOR_BUILD_NAME = 'neuron'
_MODIFICATION_TIME_FILE = 'modification_time'

if 'NRNHOME' in os.environ:
    os.environ['PATH'] += os.pathsep + \
        os.path.join(os.environ['NRNHOME'], platform.machine(), 'bin')
else:
    try:
        if os.environ['HOME'] == '/home/tclose':
            # I apologise for this little hack (this is the path on my machine,
            # to save me having to set the environment variable in eclipse)
            os.environ['PATH'] += os.pathsep + '/opt/NEURON-7.2/x86_64/bin'
    except KeyError:
        pass

def build_celltype_files(biophysics_name, nineml_path, install_dir=None,
                         build_parent_dir=None, method='derivimplicit',
                         build_mode='lazy', silent_build=False, kinetics=[]):
    """
    Generates and builds the required NMODL files for a given NCML cell class

    @param biophysics_name [str]: Name of the celltype to be built
    @param nineml_path [str]: Path to the NCML file from which the NMODL files
                              will be compiled and built
    @param install_dir [str]: Path to the directory where the NMODL files will
                              be generated and compiled
    @param build_parent_dir [str]: Used to set the default 'install_dir' path
    @param method [str]: The method option to be passed to the NeMo interpreter
                              command
    @param kinetics [list(str)]: A list of ionic components to be generated
                              using the kinetics option
    """
    (default_install_dir, params_dir,
     _, _) = get_build_paths(nineml_path, biophysics_name,
                             _SIMULATOR_BUILD_NAME,
                             build_parent_dir=build_parent_dir)
    if not install_dir:
        install_dir = default_install_dir
    if build_mode in ('force', 'build_only'):
        shutil.rmtree(install_dir, ignore_errors=True)
        shutil.rmtree(params_dir, ignore_errors=True)
    elif build_mode in ('compile_only', 'require'):
        if not os.path.exists(install_dir) or not os.path.exists(params_dir):
            raise Exception("Prebuilt installation directory '{install}'"
                            "and/or python parameters directory '{params}'"
                            "are not present, which are required for 'require'"
                            "or 'compile_only' build options"
                            .format(install=install_dir, params=params_dir))
    try:
        if not os.path.exists(install_dir):
            os.makedirs(install_dir)
        if not os.path.exists(params_dir):
            os.makedirs(params_dir)
    except IOError as e:
        raise Exception("Could not create a required neuron build directory,"
                        "check the required permissions or specify a different"
                        " parent build directory -> {}".format(e))
    # Get the stored modification time of the previous build if it exists
    install_mtime_path = os.path.join(install_dir, _MODIFICATION_TIME_FILE)
    params_mtime_path = os.path.join(params_dir, _MODIFICATION_TIME_FILE)
    if os.path.exists(install_mtime_path):
        with open(install_mtime_path) as f:
            prev_install_mtime = f.readline()
    else:
        prev_install_mtime = ''
    if os.path.exists(params_mtime_path):
        with open(params_mtime_path) as f:
            prev_params_mtime = f.readline()
    else:
        prev_params_mtime = ''
    # Get the modification time of the source NCML file for comparison with
    # the build directory
    ncml_mtime = time.ctime(os.path.getmtime(nineml_path))
    rebuilt = False
    if ((ncml_mtime != prev_install_mtime
         or ncml_mtime != prev_params_mtime) and
        build_mode != 'compile_only'):
        nemo_cmd = ("{nemo_path} {nineml_path} -p --pyparams={params}"
                    "--nmodl={output} --nmodl-method={method}"
                    "--nmodl-kinetic={kinetics}"
                    .format(nemo_path=path_to_exec('nemo'),
                            nineml_path=os.path.normpath(nineml_path),
                            output=os.path.normpath(install_dir),
                            params=params_dir, kinetics=','.join(kinetics),
                            method=method))
        try:
            sp.check_call(nemo_cmd, shell=True)
        except sp.CalledProcessError as e:
            raise Exception("Error while compiling NCML description into NMODL"
                            "code -> {}".format(e))
        # Build mode is set to 'force' because the mod files have been
        # regenerated
        with open(install_mtime_path, 'w') as f:
            f.write(ncml_mtime)
        with open(params_mtime_path, 'w') as f:
            f.write(ncml_mtime)
        rebuilt = True
    if rebuilt or build_mode == 'compile_only':
        compile_nmodl(install_dir, build_mode='force', silent=silent_build)
    # Load the parameter name translations from the params dir
    component_translations = load_component_translations(
        biophysics_name, params_dir)
    return install_dir, component_translations

def compile_nmodl(model_dir, build_mode='lazy', silent=False):
    """
    Builds all NMODL files in a directory
    @param model_dir: The path of the directory to build
    @param build_mode: Can be one of either, 'lazy', 'super_lazy', 'require',
                       'force', or 'build_only'. 'lazy' doesn't run nrnivmodl
                       if the library is found, 'require', requires that the
                       library is found otherwise throws an exception (useful
                       on clusters that require precompilation before
                       parallelisation where the error message could otherwise
                       be confusing), 'force' removes existing library if
                       found and recompiles, and 'build_only' removes existing
                       library if found, recompile and then exit
    @param verbose: Prints out verbose debugging messages
    """
    # Change working directory to model directory
    orig_dir = os.getcwd()
    try:
        os.chdir(model_dir)
    except OSError:
        raise Exception(
            "Could not find NMODL directory '{}'".format(model_dir))
    # Clean up old build directories if present
    found_required_lib = False
    for arch in BUILD_ARCHS:
        path = os.path.join(model_dir, arch)
        if os.path.exists(path):
            if build_mode in ('lazy', 'require'):
                # If the library is found, set the 'found_required_lib' flag.
                found_required_lib = True
            elif build_mode in ('force', 'build_only'):
                # Instead of flagging 'found_required_lib', the library is
                # removed to allow fresh compilation
                shutil.rmtree(path, ignore_errors=True)
    if not found_required_lib:
        if build_mode == 'require':
            raise Exception("The required NMODL binaries were not found in"
                            "directory '{}' (change the build mode from"
                            "'require' any of 'lazy', 'build_only', or"
                            "'force' in order to compile them)."
                            .format(model_dir))
        # Get platform specific command name
        if platform.system() == 'Windows':
            cmd_name = 'nrnivmodl.exe'
        else:
            cmd_name = 'nrnivmodl'
        # Check the system path for the 'nrnivmodl' command
        cmd_path = None
        for dr in os.environ['PATH'].split(os.pathsep):
            path = os.path.join(dr, cmd_name)
            if os.path.exists(path):
                cmd_path = path
                break
        if not cmd_path:
            raise Exception("Could not find nrnivmodl on the system path '{}'"
                            .format(os.environ['PATH']))
        print "Building mechanisms in '%s' directory." % model_dir
        if silent:
            with open(os.devnull, "w") as fnull:
                build_error = sp.call(cmd_path, stdout=fnull, stderr=fnull)
        else:
            # Run nrnivmodl command on directory
            build_error = sp.call(cmd_path)
        if build_error:
            raise Exception("Could not compile NMODL files in directory "
                            "'{}' - ".format(model_dir))
    elif not silent:
        print ("Found existing mechanisms in '{}' directory, compile skipped"
               " (set 'build_mode' argument to 'force' enforce recompilation"
               " them).".format(model_dir))
    os.chdir(orig_dir)
<commit_msg>added missing whitespaces to nemo command<commit_after>"""

  This module contains functions for building and loading NMODL mechanisms

  Author: Thomas G. Close (tclose@oist.jp)
  Copyright: 2012-2014 Thomas G. Close.
  License: This file is part of the "NineLine" package, which is released under
           the MIT Licence, see LICENSE for details.
"""
from __future__ import absolute_import
import os.path
import shutil
import time
import platform
import subprocess as sp

from .__init__ import (path_to_exec, get_build_paths,
                       load_component_translations)

BUILD_ARCHS = [platform.machine(), 'i686', 'x86_64', 'powerpc', 'umac']
_SIMULATOR_BUILD_NAME = 'neuron'
_MODIFICATION_TIME_FILE = 'modification_time'

if 'NRNHOME' in os.environ:
    os.environ['PATH'] += os.pathsep + \
        os.path.join(os.environ['NRNHOME'], platform.machine(), 'bin')
else:
    try:
        if os.environ['HOME'] == '/home/tclose':
            # I apologise for this little hack (this is the path on my machine,
            # to save me having to set the environment variable in eclipse)
            os.environ['PATH'] += os.pathsep + '/opt/NEURON-7.2/x86_64/bin'
    except KeyError:
        pass

def build_celltype_files(biophysics_name, nineml_path, install_dir=None,
                         build_parent_dir=None, method='derivimplicit',
                         build_mode='lazy', silent_build=False, kinetics=[]):
    """
    Generates and builds the required NMODL files for a given NCML cell class

    @param biophysics_name [str]: Name of the celltype to be built
    @param nineml_path [str]: Path to the NCML file from which the NMODL files
                              will be compiled and built
    @param install_dir [str]: Path to the directory where the NMODL files will
                              be generated and compiled
    @param build_parent_dir [str]: Used to set the default 'install_dir' path
    @param method [str]: The method option to be passed to the NeMo interpreter
                              command
    @param kinetics [list(str)]: A list of ionic components to be generated
                              using the kinetics option
    """
    (default_install_dir, params_dir,
     _, _) = get_build_paths(nineml_path, biophysics_name,
                             _SIMULATOR_BUILD_NAME,
                             build_parent_dir=build_parent_dir)
    if not install_dir:
        install_dir = default_install_dir
    if build_mode in ('force', 'build_only'):
        shutil.rmtree(install_dir, ignore_errors=True)
        shutil.rmtree(params_dir, ignore_errors=True)
    elif build_mode in ('compile_only', 'require'):
        if not os.path.exists(install_dir) or not os.path.exists(params_dir):
            raise Exception("Prebuilt installation directory '{install}'"
                            "and/or python parameters directory '{params}'"
                            "are not present, which are required for 'require'"
                            "or 'compile_only' build options"
                            .format(install=install_dir, params=params_dir))
    try:
        if not os.path.exists(install_dir):
            os.makedirs(install_dir)
        if not os.path.exists(params_dir):
            os.makedirs(params_dir)
    except IOError as e:
        raise Exception("Could not create a required neuron build directory,"
                        "check the required permissions or specify a different"
                        " parent build directory -> {}".format(e))
    # Get the stored modification time of the previous build if it exists
    install_mtime_path = os.path.join(install_dir, _MODIFICATION_TIME_FILE)
    params_mtime_path = os.path.join(params_dir, _MODIFICATION_TIME_FILE)
    if os.path.exists(install_mtime_path):
        with open(install_mtime_path) as f:
            prev_install_mtime = f.readline()
    else:
        prev_install_mtime = ''
    if os.path.exists(params_mtime_path):
        with open(params_mtime_path) as f:
            prev_params_mtime = f.readline()
    else:
        prev_params_mtime = ''
    # Get the modification time of the source NCML file for comparison with
    # the build directory
    ncml_mtime = time.ctime(os.path.getmtime(nineml_path))
    rebuilt = False
    if ((ncml_mtime != prev_install_mtime
         or ncml_mtime != prev_params_mtime) and
        build_mode != 'compile_only'):
        nemo_cmd = ("{nemo_path} {nineml_path} -p --pyparams={params} "
                    "--nmodl={output} --nmodl-method={method} "
                    "--nmodl-kinetic={kinetics}"
                    .format(nemo_path=path_to_exec('nemo'),
                            nineml_path=os.path.normpath(nineml_path),
                            output=os.path.normpath(install_dir),
                            params=params_dir, kinetics=','.join(kinetics),
                            method=method))
        try:
            sp.check_call(nemo_cmd, shell=True)
        except sp.CalledProcessError as e:
            raise Exception("Error while compiling NCML description into NMODL"
                            "code -> {}".format(e))
        # Build mode is set to 'force' because the mod files have been
        # regenerated
        with open(install_mtime_path, 'w') as f:
            f.write(ncml_mtime)
        with open(params_mtime_path, 'w') as f:
            f.write(ncml_mtime)
        rebuilt = True
    if rebuilt or build_mode == 'compile_only':
        compile_nmodl(install_dir, build_mode='force', silent=silent_build)
    # Load the parameter name translations from the params dir
    component_translations = load_component_translations(
        biophysics_name, params_dir)
    return install_dir, component_translations

def compile_nmodl(model_dir, build_mode='lazy', silent=False):
    """
    Builds all NMODL files in a directory
    @param model_dir: The path of the directory to build
    @param build_mode: Can be one of either, 'lazy', 'super_lazy', 'require',
                       'force', or 'build_only'. 'lazy' doesn't run nrnivmodl
                       if the library is found, 'require', requires that the
                       library is found otherwise throws an exception (useful
                       on clusters that require precompilation before
                       parallelisation where the error message could otherwise
                       be confusing), 'force' removes existing library if
                       found and recompiles, and 'build_only' removes existing
                       library if found, recompile and then exit
    @param verbose: Prints out verbose debugging messages
    """
    # Change working directory to model directory
    orig_dir = os.getcwd()
    try:
        os.chdir(model_dir)
    except OSError:
        raise Exception(
            "Could not find NMODL directory '{}'".format(model_dir))
    # Clean up old build directories if present
    found_required_lib = False
    for arch in BUILD_ARCHS:
        path = os.path.join(model_dir, arch)
        if os.path.exists(path):
            if build_mode in ('lazy', 'require'):
                # If the library is found, set the 'found_required_lib' flag.
                found_required_lib = True
            elif build_mode in ('force', 'build_only'):
                # Instead of flagging 'found_required_lib', the library is
                # removed to allow fresh compilation
                shutil.rmtree(path, ignore_errors=True)
    if not found_required_lib:
        if build_mode == 'require':
            raise Exception("The required NMODL binaries were not found in"
                            "directory '{}' (change the build mode from"
                            "'require' any of 'lazy', 'build_only', or"
                            "'force' in order to compile them)."
                            .format(model_dir))
        # Get platform specific command name
        if platform.system() == 'Windows':
            cmd_name = 'nrnivmodl.exe'
        else:
            cmd_name = 'nrnivmodl'
        # Check the system path for the 'nrnivmodl' command
        cmd_path = None
        for dr in os.environ['PATH'].split(os.pathsep):
            path = os.path.join(dr, cmd_name)
            if os.path.exists(path):
                cmd_path = path
                break
        if not cmd_path:
            raise Exception("Could not find nrnivmodl on the system path '{}'"
                            .format(os.environ['PATH']))
        print "Building mechanisms in '%s' directory." % model_dir
        if silent:
            with open(os.devnull, "w") as fnull:
                build_error = sp.call(cmd_path, stdout=fnull, stderr=fnull)
        else:
            # Run nrnivmodl command on directory
            build_error = sp.call(cmd_path)
        if build_error:
            raise Exception("Could not compile NMODL files in directory "
                            "'{}' - ".format(model_dir))
    elif not silent:
        print ("Found existing mechanisms in '{}' directory, compile skipped"
               " (set 'build_mode' argument to 'force' enforce recompilation"
               " them).".format(model_dir))
    os.chdir(orig_dir)
<|endoftext|>


<commit_before>#! /usr/bin/env python

# David Cournapeau
# Last Change: Tue Jun 24 04:00 PM 2008 J

# For now, just copy the tests from sandbox.pyem, so we can check that
# kmeans works OK for trivial examples.

import sys
import os.path
from scipy.testing import *

import numpy as N

from scipy.cluster.vq import kmeans, kmeans2, py_vq, py_vq2, _py_vq_1d, vq, ClusterError
try:
    from scipy.cluster import _vq
    TESTC=True
except ImportError:
    print "== Error while importing _vq, not testing C imp of vq =="
    TESTC=False

#Optional:
# import modules that are located in the same directory as this file.
DATAFILE1 = os.path.join(os.path.dirname(__file__), "data.txt")

# Global data
X   = N.array([[3.0, 3], [4, 3], [4, 2],
               [9, 2], [5, 1], [6, 2], [9, 4],
               [5, 2], [5, 4], [7, 4], [6, 5]])

CODET1  = N.array([[3.0000, 3.0000],
                   [6.2000, 4.0000],
                   [5.8000, 1.8000]])

CODET2  = N.array([[11.0/3, 8.0/3],
                   [6.7500, 4.2500],
                   [6.2500, 1.7500]])

LABEL1  = N.array([0, 1, 2, 2, 2, 2, 1, 2, 1, 1, 1])

class TestVq(TestCase):
    def test_py_vq(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        label1 = py_vq(X, initc)[0]
        assert_array_equal(label1, LABEL1)

    def test_py_vq2(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        label1 = py_vq2(X, initc)[0]
        assert_array_equal(label1, LABEL1)

    def test_vq(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        if TESTC:
            label1, dist = _vq.vq(X, initc)
            assert_array_equal(label1, LABEL1)
            tlabel1, tdist = vq(X, initc)
        else:
            print "== not testing C imp of vq =="

    #def test_py_vq_1d(self):
    #    """Test special rank 1 vq algo, python implementation."""
    #    data = X[:, 0]
    #    initc = data[:3]
    #    code = initc.copy()
    #    a, b = _py_vq_1d(data, initc)
    #    ta, tb = py_vq(data[:, N.newaxis], initc[:, N.newaxis])
    #    assert_array_equal(a, ta)
    #    assert_array_equal(b, tb)

    def test_vq_1d(self):
        """Test special rank 1 vq algo, python implementation."""
        data = X[:, 0]
        initc = data[:3]
        code = initc.copy()
        if TESTC:
            a, b = _vq.vq(data, initc)
            ta, tb = py_vq(data[:, N.newaxis], initc[:, N.newaxis])
            assert_array_equal(a, ta)
            assert_array_equal(b, tb)
        else:
            print "== not testing C imp of vq (rank 1) =="

class TestKMean(TestCase):
    def test_kmeans_simple(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        code1 = kmeans(X, code, iter = 1)[0]

        assert_array_almost_equal(code1, CODET2)

    def test_kmeans_lost_cluster(self):
        """This will cause kmean to have a cluster with no points."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))
        initk = N.array([[-1.8127404, -0.67128041],
                         [ 2.04621601, 0.07401111],
                         [-2.31149087,-0.05160469]])

        res = kmeans(data, initk)
        res = kmeans2(data, initk, missing = 'warn')
        try :
            res = kmeans2(data, initk, missing = 'raise')
            raise AssertionError("Exception not raised ! Should not happen")
        except ClusterError, e:
            print "exception raised as expected: " + str(e)

    def test_kmeans2_simple(self):
        """Testing simple call to kmeans2 and its results."""
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        code1 = kmeans2(X, code, iter = 1)[0]
        code2 = kmeans2(X, code, iter = 2)[0]

        assert_array_almost_equal(code1, CODET1)
        assert_array_almost_equal(code2, CODET2)

    def test_kmeans2_rank1(self):
        """Testing simple call to kmeans2 with rank 1 data."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))
        data1 = data[:, 0]
        data2 = data[:, 1]

        initc = data1[:3]
        code = initc.copy()
        code1 = kmeans2(data1, code, iter = 1)[0]
        code2 = kmeans2(data1, code, iter = 2)[0]

    def test_kmeans2_rank1_2(self):
        """Testing simple call to kmeans2 with rank 1 data."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))
        data1 = data[:, 0]

        code1 = kmeans2(data1, 2, iter = 1)

    def test_kmeans2_init(self):
        """Testing that kmeans2 init methods work."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))

        kmeans2(data, 3, minit = 'random')
        kmeans2(data, 3, minit = 'points')

        # Check special case 1d
        data = data[:, :1]
        kmeans2(data, 3, minit = 'random')
        kmeans2(data, 3, minit = 'points')

    def test_kmeans2_empty(self):
        """Ticket #505."""
        try:
            kmeans2([], 2)
            raise AssertionError("This should not succeed.")
        except ValueError, e:
            # OK, that's what we expect
            pass

    def test_kmeans_0k(self):
        """Regression test for #546: fail when k arg is 0."""
        try:
            kmeans(X, 0)
            raise AssertionError("kmeans with 0 clusters should fail.")
        except ValueError:
            pass

        try:
            kmeans2(X, 0)
            raise AssertionError("kmeans2 with 0 clusters should fail.")
        except ValueError:
            pass

        try:
            kmeans2(X, N.array([]))
            raise AssertionError("kmeans2 with 0 clusters should fail.")
        except ValueError:
            pass

if __name__ == "__main__":
    nose.run(argv=['', __file__])
<commit_msg>Fix useless message in test.<commit_after>#! /usr/bin/env python

# David Cournapeau
# Last Change: Tue Jun 24 04:00 PM 2008 J

# For now, just copy the tests from sandbox.pyem, so we can check that
# kmeans works OK for trivial examples.

import sys
import os.path
from scipy.testing import *

import numpy as N

from scipy.cluster.vq import kmeans, kmeans2, py_vq, py_vq2, _py_vq_1d, vq, ClusterError
try:
    from scipy.cluster import _vq
    TESTC=True
except ImportError:
    print "== Error while importing _vq, not testing C imp of vq =="
    TESTC=False

#Optional:
# import modules that are located in the same directory as this file.
DATAFILE1 = os.path.join(os.path.dirname(__file__), "data.txt")

# Global data
X   = N.array([[3.0, 3], [4, 3], [4, 2],
               [9, 2], [5, 1], [6, 2], [9, 4],
               [5, 2], [5, 4], [7, 4], [6, 5]])

CODET1  = N.array([[3.0000, 3.0000],
                   [6.2000, 4.0000],
                   [5.8000, 1.8000]])

CODET2  = N.array([[11.0/3, 8.0/3],
                   [6.7500, 4.2500],
                   [6.2500, 1.7500]])

LABEL1  = N.array([0, 1, 2, 2, 2, 2, 1, 2, 1, 1, 1])

class TestVq(TestCase):
    def test_py_vq(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        label1 = py_vq(X, initc)[0]
        assert_array_equal(label1, LABEL1)

    def test_py_vq2(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        label1 = py_vq2(X, initc)[0]
        assert_array_equal(label1, LABEL1)

    def test_vq(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        if TESTC:
            label1, dist = _vq.vq(X, initc)
            assert_array_equal(label1, LABEL1)
            tlabel1, tdist = vq(X, initc)
        else:
            print "== not testing C imp of vq =="

    #def test_py_vq_1d(self):
    #    """Test special rank 1 vq algo, python implementation."""
    #    data = X[:, 0]
    #    initc = data[:3]
    #    code = initc.copy()
    #    a, b = _py_vq_1d(data, initc)
    #    ta, tb = py_vq(data[:, N.newaxis], initc[:, N.newaxis])
    #    assert_array_equal(a, ta)
    #    assert_array_equal(b, tb)

    def test_vq_1d(self):
        """Test special rank 1 vq algo, python implementation."""
        data = X[:, 0]
        initc = data[:3]
        code = initc.copy()
        if TESTC:
            a, b = _vq.vq(data, initc)
            ta, tb = py_vq(data[:, N.newaxis], initc[:, N.newaxis])
            assert_array_equal(a, ta)
            assert_array_equal(b, tb)
        else:
            print "== not testing C imp of vq (rank 1) =="

class TestKMean(TestCase):
    def test_kmeans_simple(self):
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        code1 = kmeans(X, code, iter = 1)[0]

        assert_array_almost_equal(code1, CODET2)

    def test_kmeans_lost_cluster(self):
        """This will cause kmean to have a cluster with no points."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))
        initk = N.array([[-1.8127404, -0.67128041],
                         [ 2.04621601, 0.07401111],
                         [-2.31149087,-0.05160469]])

        res = kmeans(data, initk)
        res = kmeans2(data, initk, missing = 'warn')
        try :
            res = kmeans2(data, initk, missing = 'raise')
            raise AssertionError("Exception not raised ! Should not happen")
        except ClusterError, e:
            pass

    def test_kmeans2_simple(self):
        """Testing simple call to kmeans2 and its results."""
        initc = N.concatenate(([[X[0]], [X[1]], [X[2]]]))
        code = initc.copy()
        code1 = kmeans2(X, code, iter = 1)[0]
        code2 = kmeans2(X, code, iter = 2)[0]

        assert_array_almost_equal(code1, CODET1)
        assert_array_almost_equal(code2, CODET2)

    def test_kmeans2_rank1(self):
        """Testing simple call to kmeans2 with rank 1 data."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))
        data1 = data[:, 0]
        data2 = data[:, 1]

        initc = data1[:3]
        code = initc.copy()
        code1 = kmeans2(data1, code, iter = 1)[0]
        code2 = kmeans2(data1, code, iter = 2)[0]

    def test_kmeans2_rank1_2(self):
        """Testing simple call to kmeans2 with rank 1 data."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))
        data1 = data[:, 0]

        code1 = kmeans2(data1, 2, iter = 1)

    def test_kmeans2_init(self):
        """Testing that kmeans2 init methods work."""
        data = N.fromfile(open(DATAFILE1), sep = ", ")
        data = data.reshape((200, 2))

        kmeans2(data, 3, minit = 'random')
        kmeans2(data, 3, minit = 'points')

        # Check special case 1d
        data = data[:, :1]
        kmeans2(data, 3, minit = 'random')
        kmeans2(data, 3, minit = 'points')

    def test_kmeans2_empty(self):
        """Ticket #505."""
        try:
            kmeans2([], 2)
            raise AssertionError("This should not succeed.")
        except ValueError, e:
            # OK, that's what we expect
            pass

    def test_kmeans_0k(self):
        """Regression test for #546: fail when k arg is 0."""
        try:
            kmeans(X, 0)
            raise AssertionError("kmeans with 0 clusters should fail.")
        except ValueError:
            pass

        try:
            kmeans2(X, 0)
            raise AssertionError("kmeans2 with 0 clusters should fail.")
        except ValueError:
            pass

        try:
            kmeans2(X, N.array([]))
            raise AssertionError("kmeans2 with 0 clusters should fail.")
        except ValueError:
            pass

if __name__ == "__main__":
    nose.run(argv=['', __file__])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

from os import path
from .tools import accepted_extensions
from ..libraries.readconfig import ReadConfig
from ..platformio.project_recognition import ProjectRecognition
from .quick_menu import QuickMenu

class ProjectCheck(QuickMenu):
    """
    ProjectCheck handles the actions between sublime text and platformio.
    Before run a platformio command like initilize, compile or upload, this
    class check if the project meets the requirements to proceed with the
    command, for example if the current file has been saved, or if it's saved
    is in the src folder when the platformio sutrcture options is marked
    """
    def __init__(self):
        super(ProjectCheck, self).__init__()

        self.board_id = None
        self.port_id = None
        self.init_option = None

    def is_iot(self):
        """IOT

        Checks if the file in the current view is in the list
        of the IOT types (accepted) or not

        Returns:
            bool -- true if is in the list false if not
        """
        ext = self.get_file_extension()
        accepted = accepted_extensions()

        if(ext not in accepted):
            return False
        return True

    def is_empty(self):
        """Empty File

        Checks if the file is empty or not

        Returns:
            bool -- true is if empty
        """
        size = self.view.size()

        if(size > 0):
            return False
        return True

    def is_unsaved(self):
        """Unsaved View

        Check if the view has unsaved changes

        Returns:
            bool -- True if it's unsaved
        """
        return self.view.is_dirty()

    def check_main_requirements(self):
        """Main Requirements

        If a sketch has been never unsaved and it's not empty,
        this method will save it with a randon name based in
        the time in the temp path. If the sketch is empty it
        will not allow the command (compile/uplaod/clean) to
        continue.

        It the sketch hasn't the an IOT extension, it will be
        considerated not IOT so, you can process the file.

        If the file has unsaved changes, it will save it before
        to process the file

        Returns:
            bool -- False if any of the requirements fails.
        """
        if(self.is_empty()):
            self.dprint("not_empty_sketch")
            return False

        if(not self.get_file_name()):
            self.save_code_infile()
            self.cwd = self.get_working_project_path()

        if(not self.is_iot()):
            self.derror("not_iot_{0}", self.get_file_name())
            return False

        self.check_unsaved_changes()

        if(not path.exists(self.cwd)):
            from .tools import make_folder

            make_folder(self.cwd)

        return True

    def check_unsaved_changes(self):
        """Check unsaved changes

        Saves the changes if the view is dirty (with chages)
        """

        if(self.is_unsaved()):
            self.view.run_command('save')

    def structurize_project(self):
        """Structure Files

        If a project isn't initialized, it need to be checked
        if the open file is inside of the src folder, if it isn't
        the file need to be moved to the src folder
        """
        pio_structure = self.get_structure_option()

        if(pio_structure):
            file_path = self.get_file_path()

            dst = add_folder_to_filepath(file_path, 'src')

            if('src' not in file_path and not path.exists(dst)):
                from shutil import move
                from .tools import get_setting, save_setting

                move(file_path, dst)
                self.view.retarget(dst)

                if(get_setting('freeze_sketch', None)):
                    save_setting('freeze_sketch', dst)

    def override_src(self, wipe=False):
        """Adds src_dir

        When you don't want to keep the platformio file structure, you need to add
        the 'src_dir' flag in the platformio.ini with the path of your sketch/project.
        Here we add that option when platformio structure is not enabled
        """
        if(self.is_native()):
            return

        write_file = False
        current_src = None
        platformio_head = 'platformio'
        pio_structure = self.get_structure_option()
        project_path = self.get_project_path()
        ini_path = self.get_ini_path()

        config = ReadConfig()
        config.read(ini_path)

        # get string if exists
        if(config.has_option(platformio_head, 'src_dir')):
            current_src = config.get(platformio_head, 'src_dir')[0]

        # remove option
        if(wipe and current_src):
            config.remove_option(platformio_head, 'src_dir')

            if(not config.options(platformio_head)):
                config.remove_section(platformio_head)
            write_file = True

        # check section
        if(not config.has_section(platformio_head)):
            config.add_section(platformio_head)
            write_file = True

        # change only in case it's different
        if(project_path != current_src):
            config.set(platformio_head, 'src_dir', project_path)
            write_file = True

        if(write_file):
            with open(ini_path, 'w') as configfile:
                config.write(configfile)

    def close_file(self):
        """Close File Window

        Close the current focused windows in sublime text
        """
        self.window.run_command('close_file')

    def check_board_selected(self):
        """Checks Board Selection

        If an environment is stored in the preferences file, it will
        be loaded in the board_id object, if not, it will show the
        quick panel to select the board
        """
        self.board_id = self.get_environment()

        if(not self.board_id):
            selected_boards = self.get_selected_boards()

            if(not selected_boards):
                QuickMenu().quick_boards()
                return

            QuickMenu().quick_environments()
            return

    def check_port_selected(self):
        """Checks Serial Port Selection

        If the serial port is stored in the preferences file, it will
        be loaded in the port_id object, if not, it will show the
        quick panel to select the port
        """
        from re import search

        self.port_id = self.get_serial_port()
        ports_list = self.get_ports_list()

        port_ready = [port[1] for port in ports_list if self.port_id == port[1]]
        ip_device = search(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", self.port_id) if self.port_id else None

        if(not port_ready and ip_device is None):
            QuickMenu().quick_serial_ports()
            self.port_id = None

    def check_serial_monitor(self):
        """Check monitor serial

        Checks if the monitor serial is currently running
        and close it.

        It will also stores a reference in the preferences (last_action)
        to run the serial monitor next time
        """
        from . import serial
        from .tools import save_setting

        port_id = self.port_id

        if(port_id in serial.serials_in_use):
            serial_monitor = serial.get_serial_monitor(port_id)

            serial_monitor.stop()
            del serial.serial_monitor_dict[port_id]

            save_setting('run_monitor', True)

    def check_auth_ota(self):
        """Check auth

        Checks if the selected port is a mdns service, and if it needs
        authentification to upload the sketch

        Returns:
            bool -- None when not auth, false when none pass is stored
        """

        from .tools import get_setting
        from re import search

        ended = True

        platform = self.get_platform()
        ip_device = search(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", self.port_id)

        if(platform and 'espressif' not in platform and ip_device is not None):
            return False

        if(platform and 'espressif' not in platform):
            return ended

        auth = None
        ini_path = self.get_ini_path()
        config = ReadConfig()
        config.read(ini_path)

        ports_list = self.get_ports_list()

        for port in ports_list:
            if(self.port_id == port[1]):
                try:
                    auth = port[2]
                    break
                except:
                    return ended

        environment = 'env:{0}'.format(self.board_id)
        auth_pass = get_setting('auth_pass', None)

        if(auth == 'no'):
            if(not auth_pass):
                if(config.has_option(environment, 'upload_flags')):
                    config.remove_option(environment, 'upload_flags')

                    with open(ini_path, 'w') as configfile:
                        config.write(configfile)
            return ended

        if(auth == 'yes' and not auth_pass):
            from .tools import save_sysetting
            self.window.run_command("deviot_set_password")
            save_sysetting('last_action', 3)
            return ended

        flag = '--auth={0}'.format(auth_pass)
        config.set(environment, 'upload_flags', flag)

        with open(ini_path, 'w') as configfile:
            config.write(configfile)

        return ended

    def save_code_infile(self):
        """Save Code

        If the sketch in the current view has been not saved, it generate
        a random name and stores in a temp folder.

        Arguments: view {ST Object} -- Object with multiples options of ST
        """
        from os import path
        from time import time
        from sublime import Region
        from ..libraries.file import File
        from ..libraries.tools import make_folder

        ext = '.ino'

        tmppath = self.get_temp_path()
        filename = str(time()).split('.')[0]
        filepath = path.join(tmppath, filename, 'src')

        make_folder(filepath)

        fullpath = filename + ext
        fullpath = path.join(filepath, fullpath)

        region = Region(0, self.view.size())
        text = self.view.substr(region)
        file = File(fullpath)
        file.write(text)

        self.view.set_scratch(True)
        self.window.run_command('close')
        self.view = self.window.open_file(fullpath)

def add_folder_to_filepath(src_path, new_folder):
    """Add folder

    Add a new folder at the end of the given specific path

    Arguments:
        src_path {str} -- initial path including the filename
        new_folder {str} -- string to add after the last folder in the path

    Returns:
        str -- file path with the new folder added
    """

    folder = path.dirname(src_path)
    file_name = path.basename(src_path)
    new_path = path.join(folder, new_folder, file_name)

    return new_path<commit_msg>Bug fixes introduced in: https://github.com/gepd/Deviot/commit/7e4d6598d51bc2dad9277dd67ac4ce00c8eba6d7<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

from os import path
from .tools import accepted_extensions
from ..libraries.readconfig import ReadConfig
from ..platformio.project_recognition import ProjectRecognition
from .quick_menu import QuickMenu

class ProjectCheck(QuickMenu):
    """
    ProjectCheck handles the actions between sublime text and platformio.
    Before run a platformio command like initilize, compile or upload, this
    class check if the project meets the requirements to proceed with the
    command, for example if the current file has been saved, or if it's saved
    is in the src folder when the platformio sutrcture options is marked
    """
    def __init__(self):
        super(ProjectCheck, self).__init__()

        self.board_id = None
        self.port_id = None
        self.init_option = None

    def is_iot(self):
        """IOT

        Checks if the file in the current view is in the list
        of the IOT types (accepted) or not

        Returns:
            bool -- true if is in the list false if not
        """
        ext = self.get_file_extension()
        accepted = accepted_extensions()

        if(ext not in accepted):
            return False
        return True

    def is_empty(self):
        """Empty File

        Checks if the file is empty or not

        Returns:
            bool -- true is if empty
        """
        size = self.view.size()

        if(size > 0):
            return False
        return True

    def is_unsaved(self):
        """Unsaved View

        Check if the view has unsaved changes

        Returns:
            bool -- True if it's unsaved
        """
        return self.view.is_dirty()

    def check_main_requirements(self):
        """Main Requirements

        If a sketch has been never unsaved and it's not empty,
        this method will save it with a randon name based in
        the time in the temp path. If the sketch is empty it
        will not allow the command (compile/uplaod/clean) to
        continue.

        It the sketch hasn't the an IOT extension, it will be
        considerated not IOT so, you can process the file.

        If the file has unsaved changes, it will save it before
        to process the file

        Returns:
            bool -- False if any of the requirements fails.
        """
        if(self.is_empty()):
            self.dprint("not_empty_sketch")
            return False

        if(not self.get_file_name()):
            self.save_code_infile()
            self.cwd = self.get_working_project_path()

        if(not self.is_iot()):
            self.derror("not_iot_{0}", self.get_file_name())
            return False

        self.check_unsaved_changes()

        if(not path.exists(self.cwd)):
            from .tools import make_folder

            make_folder(self.cwd)

        return True

    def check_unsaved_changes(self):
        """Check unsaved changes

        Saves the changes if the view is dirty (with chages)
        """

        if(self.is_unsaved()):
            self.view.run_command('save')

    def structurize_project(self):
        """Structure Files

        If a project isn't initialized, it need to be checked
        if the open file is inside of the src folder, if it isn't
        the file need to be moved to the src folder
        """
        pio_structure = self.get_structure_option()

        if(pio_structure):
            file_path = self.get_file_path()

            dst = add_folder_to_filepath(file_path, 'src')

            if('src' not in file_path and not path.exists(dst)):
                from shutil import move
                from .tools import get_setting, save_setting

                move(file_path, dst)
                self.view.retarget(dst)

                if(get_setting('freeze_sketch', None)):
                    save_setting('freeze_sketch', dst)

    def override_src(self, wipe=False):
        """Adds src_dir

        When you don't want to keep the platformio file structure, you need to add
        the 'src_dir' flag in the platformio.ini with the path of your sketch/project.
        Here we add that option when platformio structure is not enabled
        """
        if(self.is_native()):
            return

        write_file = False
        current_src = None
        platformio_head = 'platformio'
        pio_structure = self.get_structure_option()
        project_path = self.get_project_path()
        ini_path = self.get_ini_path()

        config = ReadConfig()
        config.read(ini_path)

        # get string if exists
        if(config.has_option(platformio_head, 'src_dir')):
            current_src = config.get(platformio_head, 'src_dir')[0]

        # remove option
        if(wipe and current_src):
            config.remove_option(platformio_head, 'src_dir')

            if(not config.options(platformio_head)):
                config.remove_section(platformio_head)
            write_file = True

        # check section
        if(not config.has_section(platformio_head)):
            config.add_section(platformio_head)
            write_file = True

        # change only in case it's different
        if(project_path != current_src):
            config.set(platformio_head, 'src_dir', project_path)
            write_file = True

        if(write_file):
            with open(ini_path, 'w') as configfile:
                config.write(configfile)

    def close_file(self):
        """Close File Window

        Close the current focused windows in sublime text
        """
        self.window.run_command('close_file')

    def check_board_selected(self):
        """Checks Board Selection

        If an environment is stored in the preferences file, it will
        be loaded in the board_id object, if not, it will show the
        quick panel to select the board
        """
        self.board_id = self.get_environment()

        if(not self.board_id):
            selected_boards = self.get_selected_boards()

            if(not selected_boards):
                self.window.run_command('deviot_select_boards')
                return

            self.window.run_command('deviot_select_environment')
            return

    def check_port_selected(self):
        """Checks Serial Port Selection

        If the serial port is stored in the preferences file, it will
        be loaded in the port_id object, if not, it will show the
        quick panel to select the port
        """
        from re import search

        self.port_id = self.get_serial_port()
        ports_list = self.get_ports_list()

        port_ready = [port[1] for port in ports_list if self.port_id == port[1]]
        ip_device = search(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", self.port_id) if self.port_id else None

        if(not port_ready and ip_device is None):
            self.window.run_command('deviot_select_port')
            self.port_id = None

    def check_serial_monitor(self):
        """Check monitor serial

        Checks if the monitor serial is currently running
        and close it.

        It will also stores a reference in the preferences (last_action)
        to run the serial monitor next time
        """
        from . import serial
        from .tools import save_setting

        port_id = self.port_id

        if(port_id in serial.serials_in_use):
            serial_monitor = serial.get_serial_monitor(port_id)

            serial_monitor.stop()
            del serial.serial_monitor_dict[port_id]

            save_setting('run_monitor', True)

    def check_auth_ota(self):
        """Check auth

        Checks if the selected port is a mdns service, and if it needs
        authentification to upload the sketch

        Returns:
            bool -- None when not auth, false when none pass is stored
        """

        from .tools import get_setting
        from re import search

        ended = True

        platform = self.get_platform()
        ip_device = search(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", self.port_id)

        if(platform and 'espressif' not in platform and ip_device is not None):
            return False

        if(platform and 'espressif' not in platform):
            return ended

        auth = None
        ini_path = self.get_ini_path()
        config = ReadConfig()
        config.read(ini_path)

        ports_list = self.get_ports_list()

        for port in ports_list:
            if(self.port_id == port[1]):
                try:
                    auth = port[2]
                    break
                except:
                    return ended

        environment = 'env:{0}'.format(self.board_id)
        auth_pass = get_setting('auth_pass', None)

        if(auth == 'no'):
            if(not auth_pass):
                if(config.has_option(environment, 'upload_flags')):
                    config.remove_option(environment, 'upload_flags')

                    with open(ini_path, 'w') as configfile:
                        config.write(configfile)
            return ended

        if(auth == 'yes' and not auth_pass):
            from .tools import save_sysetting
            self.window.run_command("deviot_set_password")
            save_sysetting('last_action', 3)
            return ended

        flag = '--auth={0}'.format(auth_pass)
        config.set(environment, 'upload_flags', flag)

        with open(ini_path, 'w') as configfile:
            config.write(configfile)

        return ended

    def save_code_infile(self):
        """Save Code

        If the sketch in the current view has been not saved, it generate
        a random name and stores in a temp folder.

        Arguments: view {ST Object} -- Object with multiples options of ST
        """
        from os import path
        from time import time
        from sublime import Region
        from ..libraries.file import File
        from ..libraries.tools import make_folder

        ext = '.ino'

        tmppath = self.get_temp_path()
        filename = str(time()).split('.')[0]
        filepath = path.join(tmppath, filename, 'src')

        make_folder(filepath)

        fullpath = filename + ext
        fullpath = path.join(filepath, fullpath)

        region = Region(0, self.view.size())
        text = self.view.substr(region)
        file = File(fullpath)
        file.write(text)

        self.view.set_scratch(True)
        self.window.run_command('close')
        self.view = self.window.open_file(fullpath)

def add_folder_to_filepath(src_path, new_folder):
    """Add folder

    Add a new folder at the end of the given specific path

    Arguments:
        src_path {str} -- initial path including the filename
        new_folder {str} -- string to add after the last folder in the path

    Returns:
        str -- file path with the new folder added
    """

    folder = path.dirname(src_path)
    file_name = path.basename(src_path)
    new_path = path.join(folder, new_folder, file_name)

    return new_path<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

from aiohttp import web
import json
import sys, traceback

__author__ = "Praveen Garg"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

def json_error(message):
    return web.Response(body=json.dumps({'error': message}).encode('utf-8'), content_type='application/json')

async def error_middleware(app, handler):
    async def middleware_handler(request):
        if_trace = request.query.get('trace') if 'trace' in request.query and request.query.get('trace') == '1' else None

        try:
            response = await handler(request)
            if response.status == 404:
                return handle_api_exception({"code": response.status, "message": response.message}, if_trace)
            return response
        except (web.HTTPNotFound, web.HTTPBadRequest) as ex:
            return handle_api_exception({"code": ex.status_code, "message": ex.reason}, if_trace)
        except web.HTTPException as ex:
            raise
        # Below Exception must come last as it is the super class of all exceptions
        except Exception as ex:
            return handle_api_exception(ex, if_trace)

    return middleware_handler

def handle_api_exception(ex, if_trace=0):
    if not isinstance(ex, Exception):
        err_msg = ex
    else:
        _class = ex.__class__.__name__
        _msg = str(ex)

        scode = 500
        err_msg = {"code": scode, "message": '['+_class+']'+_msg}

    if if_trace:
        err_msg.update({"exception": _class, "traceback": traceback.format_exc()})

    return json_error(err_msg)
<commit_msg>Minor fix<commit_after># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

from aiohttp import web
import json
import sys, traceback

__author__ = "Praveen Garg"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

def json_error(message):
    return web.Response(body=json.dumps({'error': message}).encode('utf-8'), content_type='application/json')

async def error_middleware(app, handler):
    async def middleware_handler(request):
        if_trace = request.query.get('trace') if 'trace' in request.query and request.query.get('trace') == '1' else None

        try:
            response = await handler(request)
            if response.status == 404:
                return handle_api_exception({"code": response.status, "message": response.message}, ex.__class__.__name__, if_trace)
            return response
        except (web.HTTPNotFound, web.HTTPBadRequest) as ex:
            return handle_api_exception({"code": ex.status_code, "message": ex.reason}, ex.__class__.__name__, if_trace)
        except web.HTTPException as ex:
            raise
        # Below Exception must come last as it is the super class of all exceptions
        except Exception as ex:
            return handle_api_exception(ex, ex.__class__.__name__, if_trace)

    return middleware_handler

def handle_api_exception(ex, _class=None, if_trace=0):
    if not isinstance(ex, Exception):
        err_msg = ex
    else:
        _msg = str(ex)

        scode = 500
        err_msg = {"code": scode, "message": '['+_class+']'+_msg}

    if if_trace:
        err_msg.update({"exception": _class, "traceback": traceback.format_exc()})

    return json_error(err_msg)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
def foo():
    r = some_function()
    if r == (-1):
        return(-1)
    return r
def bar():
    r = foo()
    if r == (-1):
        print('Error')
    else:
        pass
#一旦出错，还要一级一级上报，除此之外，可以使用try..except..finall
try:
    print('try...')
    r = 10 / 2
    print('result',r)
except ZeroDivisionError as e:
    print('except:',e)
finally:
    print('finally...')
print('END')
#当某些代码执行有可能发生错误的时候，可以用try运行该类代码，此时一旦try当中遇到错误，则跳转到except后面代码执行
#然后在执行finally后面的代码，无论是否执行except，都会执行finally后面代码
try:
    print('try...')
    r = 10 / int('a')
    print('result:',r)
except ValueError as e:
    print('ValueError:',e)
except ZeroDivisionError as e:
    print('ZeroDivisionError:',e)
else:print('no error')
finally:
    print('finally...')
print('END')
#可以使用多个except来执行错误发现，如上
def foo(s):
    return 10 / int(s)
def bar(s):
    return foo(s)*2
def main():
    bar('0')
#    try:
#        print(bar('0'))
#    except Exception as e:
#        print('Error:',e)
#    finally:
#        print('finally...')
#main()
#对于多层调用的情况，只需要在调用层进行捕获操作，即可确认问题出现
#如果不用try的话，错误会一直向上抛，直到最终错误出现的代码，可以确认错误从哪里出现
#但是一旦出现错误的话，程序就会异常结束，这样没意义，应当让python打印出错误来
import logging
def foo(s):
    return 10 / int(s)
def bar(s):
    return foo(s)*2
def main():
    try:
        bar('0')
    except Exception as e:
        logging.exception(e)
main()
print('END')
#通过日志记录，可以让问题打印，但是不暂停执行程序，程序正常执行完毕之后才会退出
#抛出错误可以自定义错误的类，然后执行
class FooError(ValueError):
    pass
def foo(s):
    n = int(s)
    if n == 0:
        raise FooError('invalid value:%s' % s)
    return 10 / n
foo('0')
#可以从ValueError中派生一个自定义类，然后从该类当中抛出错误
def foo(s):
    n = int(s)
    if n == 0:
        raise ValueError('invalid value: %s' % s)
    return 10 / n
def bar():
    try:
        foo('0')
    except ValueError as e:
        print('ValueError!')
        raise
bar()
#使用raise的话，错误会抛出，由foo内部的错误进行显示
<commit_msg>20170228<commit_after># -*- coding: utf-8 -*-
def foo():
    r = some_function()
    if r == (-1):
        return(-1)
    return r
def bar():
    r = foo()
    if r == (-1):
        print('Error')
    else:
        pass
#一旦出错，还要一级一级上报，除此之外，可以使用try..except..finall
try:
    print('try...')
    r = 10 / 0
    print('result',r)
except ZeroDivisionError as e:
    print('except:',e)
finally:
    print('finally...')
print('END')
#当某些代码执行有可能发生错误的时候，可以用try运行该类代码，此时一旦try当中遇到错误，则跳转到except后面代码执行
#然后在执行finally后面的代码，无论是否执行except，都会执行finally后面代码
try:
    print('try...')
    r = 10 / int('a')
    print('result:',r)
except ValueError as e:
    print('ValueError:',e)
except ZeroDivisionError as e:
    print('ZeroDivisionError:',e)
else:print('no error')
finally:
    print('finally...')
print('END')
#可以使用多个except来执行错误发现，如上
def foo(s):
    return 10 / int(s)
def bar(s):
    return foo(s)*2
def main():
    bar('0')
#    try:
#        print(bar('0'))
#    except Exception as e:
#        print('Error:',e)
#    finally:
#        print('finally...')
#main()
#对于多层调用的情况，只需要在调用层进行捕获操作，即可确认问题出现
#如果不用try的话，错误会一直向上抛，直到最终错误出现的代码，可以确认错误从哪里出现
#但是一旦出现错误的话，程序就会异常结束，这样没意义，应当让python打印出错误来
import logging
def foo(s):
    return 10 / int(s)
def bar(s):
    return foo(s)*2
def main():
    try:
        bar('0')
    except Exception as e:
        logging.exception(e)
main()
print('END')
#通过日志记录，可以让问题打印，但是不暂停执行程序，程序正常执行完毕之后才会退出
#抛出错误可以自定义错误的类，然后执行
class FooError(ValueError):
    pass
def foo(s):
    n = int(s)
    if n == 0:
        raise FooError('invalid value:%s' % s)
    return 10 / n
foo('0')
#可以从ValueError中派生一个自定义类，然后从该类当中抛出错误
def foo(s):
    n = int(s)
    if n == 0:
        raise ValueError('invalid value: %s' % s)
    return 10 / n
def bar():
    try:
        foo('0')
    except ValueError as e:
        print('ValueError!')
        raise
bar()
#使用raise的话，错误会抛出，由foo内部的错误进行显示
<|endoftext|>


<commit_before>__version__ = '4.8.0'
<commit_msg>Bump version to 4.8.1<commit_after>__version__ = '4.8.1'
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

import os
import logging
import time
import math
import multiprocessing
import breakmer.processor.target as target
import breakmer.utils as utils

__author__ = "Ryan Abo"
__copyright__ = "Copyright 2015, Ryan Abo"
__email__ = "ryanabo@gmail.com"
__license__ = "MIT"

def analyze_targets(targetList):
    """Analyze a list of targets.
    A list of TargetManager objects are passed in to be analyzed independently.
    Each target ref data is set, if necessary, then the reads are extracted,
    contigs built, and calls made.
    Args:
        targetList: A list of TargetManager objects, representing target regions.
    Returns:
        None
    """

    for targetRegion in targetList:
        utils.log('breakmer.processor.analysis', 'info', 'Analyzing %s' % targetRegion.name)
        targetRegion.set_ref_data()

        # If only presetting data stop here.
        if targetRegion.params.get_param('preset_ref_data'):
            continue
        # targetRegion fails to find any interesting reads to use. Exiting.
        if not targetRegion.find_sv_reads():
            continue
        targetRegion.compare_kmers()
        # targetRegion.resolve_sv()
        # targetRegion.complete_analysis()

class RunTracker:
    """Class to manage the running of all the target region analyses.
    The params object is passed in with all the input information.
    The run() function creates the target region objects from the
    param inputs and then starts the analysis for each target.
    Args:
        params: ParamManager object.
    Returns:
        None
    """
    def __init__(self, params):
        self.params = params
        self.results = []
        self.targets = {}
        self.summary = {}
        self.summary_header = ''
        self.logger = logging.getLogger('breakmer.processor.analysis')

    def run(self):
        """Create and analyze the target regions.
        The target objects are made and grouped for multiprocessing (if set)
        and these are all analyzed independently. This is where the analysis
        starts and ends.
        Args:
            None
        Returns
            None
        """
        startTime = time.clock()
        targetAnalysisList = self.create_targets()

        if not self.params.get_param('preset_ref_data'):
            self.params.start_blat_server()

        nprocs = int(self.params.get_param('nprocs'))
        if nprocs > 1:
            self.logger.info('Creating all reference data.')
            p = multiprocessing.Pool(nprocs)
            p.map(analyze_targets, targetAnalysisList)
        else:
            analyze_targets(targetAnalysisList)

#        self.write_output()
        self.logger.info('Analysis complete in %s' % str(time.clock() - startTime))

        if not self.params.get_param('keep_blat_server'):
            cmd = '%s stop localhost %d' % (self.params.opts['gfserver'], int(self.params.get_param('blat_port')))
            os.system(cmd)
        print 'Analysis complete!'

    def create_targets(self):
        """Create target objects and group them by the number of
        multiprocs that are specified (i.e. n=1 for 1 processor.)
        Args:
            None
        Returns:
            trgtGroups: List of lists of target objects. Each list is
                        analyzed by a processor.
        """
        nprocs = int(self.params.get_param('nprocs'))
        multiprocs = nprocs > 1
        ngroups = nprocs
        ntargets = len(self.params.targets)
        ntargetsPerGroup = ntargets / nprocs
        modval = math.fmod(ntargets, nprocs)
        if modval > 0:
            ngroups += 1
        trgtGroups = []
        trgtGroup = []

        targetNames = self.params.targets.keys()
        targetNames.sort()
        for targetName in targetNames:
            trgt = target.TargetManager(targetName, self.params)
            self.targets[targetName] = trgt
            if multiprocs:
                if len(trgtGroup) == ntargetsPerGroup:
                    trgtGroups.append(trgtGroup)
                    trgtGroup = []
                trgtGroup.append(trgt)
            else:
                trgtGroups.append(trgt)

        if multiprocs:
            if len(trgtGroup) < ntargetsPerGroup:
                trgtGroups[-1].extend(trgtGroup)
            else:
                trgtGroups.append(trgtGroup)
        return trgtGroups

    def write_output(self):
        """Write the information for all results to file.
        Header is written at the top of the file if option to remove is not
        specified.
        Args:
            None
        Returns:
            None
        """

        resultFiles = {}
        for res in self.results:
            tag = res[6]
            if tag not in resultFiles:
                header = "\t".join(['genes', 'target_breakpoints', 'align_cigar', 'mismatches', 'strands', 'rep_overlap_segment_len', 'sv_type', 'split_read_count', 'nkmers', 'disc_read_count', 'breakpoint_coverages', 'contig_id', 'contig_seq']) + "\n"
                res_fn = os.path.join(self.params.paths['output'], self.params.opts['analysis_name'] + "_" + tag + "_svs.out")
                self.logger.info('Writing %s output file %s' % (tag, res_fn))
                resultFiles[tag] = open(res_fn, 'w')
                if not self.params.opts['no_output_header']:
                    resultFiles[tag].write(header)
            resultFiles[tag].write("\t".join([str(x) for x in res]) + "\n")

        for f in resultFiles:
            resultFiles[f].close()

#        summary_fn = os.path.join(self.params.paths['output'], self.params.opts['analysis_name']+"_summary.out")
#        summary_f = open(summary_fn,'w')
#        self.logger.info('Writing summary file to %s'%summary_fn)
#        summary_f.write(self.summary_header+"\n")
#        keys = self.summary.keys()
#        keys.sort()
#        for gene in keys :
#            summary_f.write(self.summary[gene]+"\n")
#        summary_f.close()
<commit_msg>Bug fixes, refactoring.<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

import os
import logging
import time
import math
import multiprocessing
import breakmer.processor.target as target
import breakmer.utils as utils

__author__ = "Ryan Abo"
__copyright__ = "Copyright 2015, Ryan Abo"
__email__ = "ryanabo@gmail.com"
__license__ = "MIT"

def analyze_targets(targetList):
    """Analyze a list of targets.
    A list of TargetManager objects are passed in to be analyzed independently.
    Each target ref data is set, if necessary, then the reads are extracted,
    contigs built, and calls made.
    Args:
        targetList: A list of TargetManager objects, representing target regions.
    Returns:
        None
    """

    for targetRegion in targetList:
        utils.log('breakmer.processor.analysis', 'info', 'Analyzing %s' % targetRegion.name)
        targetRegion.set_ref_data()

        # If only presetting data stop here.
        if targetRegion.params.get_param('preset_ref_data'):
            continue
        # targetRegion fails to find any interesting reads to use. Exiting.
        if not targetRegion.find_sv_reads():
            continue
        targetRegion.compare_kmers()
        targetRegion.resolve_sv()
        # targetRegion.complete_analysis()

class RunTracker:
    """Class to manage the running of all the target region analyses.
    The params object is passed in with all the input information.
    The run() function creates the target region objects from the
    param inputs and then starts the analysis for each target.
    Args:
        params: ParamManager object.
    Returns:
        None
    """
    def __init__(self, params):
        self.params = params
        self.results = []
        self.targets = {}
        self.summary = {}
        self.summary_header = ''
        self.logger = logging.getLogger('breakmer.processor.analysis')

    def run(self):
        """Create and analyze the target regions.
        The target objects are made and grouped for multiprocessing (if set)
        and these are all analyzed independently. This is where the analysis
        starts and ends.
        Args:
            None
        Returns
            None
        """
        startTime = time.clock()
        targetAnalysisList = self.create_targets()

        if not self.params.get_param('preset_ref_data'):
            self.params.start_blat_server()

        nprocs = int(self.params.get_param('nprocs'))
        if nprocs > 1:
            self.logger.info('Creating all reference data.')
            p = multiprocessing.Pool(nprocs)
            p.map(analyze_targets, targetAnalysisList)
        else:
            analyze_targets(targetAnalysisList)

#        self.write_output()
        self.logger.info('Analysis complete in %s' % str(time.clock() - startTime))

        if not self.params.get_param('keep_blat_server'):
            cmd = '%s stop localhost %d' % (self.params.opts['gfserver'], int(self.params.get_param('blat_port')))
            os.system(cmd)
        print 'Analysis complete!'

    def create_targets(self):
        """Create target objects and group them by the number of
        multiprocs that are specified (i.e. n=1 for 1 processor.)
        Args:
            None
        Returns:
            trgtGroups: List of lists of target objects. Each list is
                        analyzed by a processor.
        """
        nprocs = int(self.params.get_param('nprocs'))
        multiprocs = nprocs > 1
        ngroups = nprocs
        ntargets = len(self.params.targets)
        ntargetsPerGroup = ntargets / nprocs
        modval = math.fmod(ntargets, nprocs)
        if modval > 0:
            ngroups += 1
        trgtGroups = []
        trgtGroup = []

        targetNames = self.params.targets.keys()
        targetNames.sort()
        for targetName in targetNames:
            trgt = target.TargetManager(targetName, self.params)
            self.targets[targetName] = trgt
            if multiprocs:
                if len(trgtGroup) == ntargetsPerGroup:
                    trgtGroups.append(trgtGroup)
                    trgtGroup = []
                trgtGroup.append(trgt)
            else:
                trgtGroups.append(trgt)

        if multiprocs:
            if len(trgtGroup) < ntargetsPerGroup:
                trgtGroups[-1].extend(trgtGroup)
            else:
                trgtGroups.append(trgtGroup)
        return trgtGroups

    def write_output(self):
        """Write the information for all results to file.
        Header is written at the top of the file if option to remove is not
        specified.
        Args:
            None
        Returns:
            None
        """

        resultFiles = {}
        for res in self.results:
            tag = res[6]
            if tag not in resultFiles:
                header = "\t".join(['genes', 'target_breakpoints', 'align_cigar', 'mismatches', 'strands', 'rep_overlap_segment_len', 'sv_type', 'split_read_count', 'nkmers', 'disc_read_count', 'breakpoint_coverages', 'contig_id', 'contig_seq']) + "\n"
                res_fn = os.path.join(self.params.paths['output'], self.params.opts['analysis_name'] + "_" + tag + "_svs.out")
                self.logger.info('Writing %s output file %s' % (tag, res_fn))
                resultFiles[tag] = open(res_fn, 'w')
                if not self.params.opts['no_output_header']:
                    resultFiles[tag].write(header)
            resultFiles[tag].write("\t".join([str(x) for x in res]) + "\n")

        for f in resultFiles:
            resultFiles[f].close()

#        summary_fn = os.path.join(self.params.paths['output'], self.params.opts['analysis_name']+"_summary.out")
#        summary_f = open(summary_fn,'w')
#        self.logger.info('Writing summary file to %s'%summary_fn)
#        summary_f.write(self.summary_header+"\n")
#        keys = self.summary.keys()
#        keys.sort()
#        for gene in keys :
#            summary_f.write(self.summary[gene]+"\n")
#        summary_f.close()
<|endoftext|>


<commit_before>import unicodedata

class TweetImporter(object):
    def __init__(self, twitter_client, database):
        self.twitter_client = twitter_client
        self.database = database

    def importData(self, user, tweets_number=10):
        followers = self.twitter_client.followers_list(screen_name=user, count=10)

        id = 0
        for j, follower in enumerate(followers):
            tweets = self.twitter_client.user_timeline(screen_name=follower, count=tweets_number)
            mentions_list = []
            for i, status in enumerate(tweets):
                id += 1
                tweet = status._json
                text = tweet['text']
                date = tweet['created_at']
                entities = tweet['entities']
                user_mentions = entities['user_mentions']

                if len(user_mentions) > 0:
                    for mention in user_mentions:
                        mentions_list.append(mention['screen_name'])

                text_string = unicodedata.normalize('NFKD', text).encode('ascii','ignore')
                date_string = unicodedata.normalize('NFKD', date).encode('ascii','ignore')
                name_mentions_string = ",".join(mentions_list)

                self.database.feed_table(
                    follower,
                    text_string,
                    date_string,
                    name_mentions_string)
<commit_msg>Fix: Mentions per user were incremented all the time<commit_after>import unicodedata

class TweetImporter(object):
    def __init__(self, twitter_client, database):
        self.twitter_client = twitter_client
        self.database = database

    def importData(self, user, tweets_number=10):
        followers = self.twitter_client.followers_list(screen_name=user, count=10)

        id = 0
        for j, follower in enumerate(followers):
            tweets = self.twitter_client.user_timeline(screen_name=follower, count=tweets_number)
            for i, status in enumerate(tweets):
                id += 1
                tweet = status._json
                text = tweet['text']
                date = tweet['created_at']
                entities = tweet['entities']
                user_mentions = entities['user_mentions']
                mentions_list = []

                if len(user_mentions) > 0:
                    for mention in user_mentions:
                        mentions_list.append(mention['screen_name'])

                text_string = unicodedata.normalize('NFKD', text).encode('ascii','ignore')
                date_string = unicodedata.normalize('NFKD', date).encode('ascii','ignore')
                name_mentions_string = ",".join(mentions_list)

                self.database.feed_table(
                    follower,
                    text_string,
                    date_string,
                    name_mentions_string)
<|endoftext|>