<commit_before>import json
from json.decoder import JSONDecodeError
from urllib.parse import urlencode
from settings import JIRA_API_URL
from tools import make_basic_auth_str, make_http_request

class JiraApi:
    def __init__(self, login, password):
        self.authorization = make_basic_auth_str(login, password)

    def get_headers(self):
        return {
            'Authorization': 'Basic %s' % self.authorization,
            'Content-Type': 'application/json'
        }

    def get_my_permissions(self):
        url = JIRA_API_URL + 'mypermissions'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            data = json.loads(result)
            permissions = data['permissions']
        except (JSONDecodeError, KeyError, TypeError):
            raise JiraError
        else:
            return permissions

    def get_projects(self):
        url = JIRA_API_URL + 'project'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            return json.loads(result)
        except JSONDecodeError:
            raise JiraError

    def get_project_versions(self, project_key):
        url = JIRA_API_URL + 'project/' + project_key + '/versions'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            return json.loads(result)
        except JSONDecodeError:
            raise JiraError

    def get_issues(self, jql):
        url = JIRA_API_URL + 'search'
        query = urlencode({
            'jql': jql
        })

        result = make_http_request(url + '?' + query, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            data = json.loads(result)
            issues = data['issues']
        except (JSONDecodeError, KeyError):
            raise JiraError
        else:
            return issues

    def get_issue(self, key):
        url = JIRA_API_URL + 'issue/' + key

        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            issue = json.loads(result)
        except JSONDecodeError:
            raise JiraError
        else:
            return issue

class JiraError(Exception):
    pass

class JiraConnectionError(JiraError):
    pass
<commit_msg>Set max results for jira tasks<commit_after>import json
from json.decoder import JSONDecodeError
from urllib.parse import urlencode
from settings import JIRA_API_URL
from tools import make_basic_auth_str, make_http_request

class JiraApi:
    def __init__(self, login, password):
        self.authorization = make_basic_auth_str(login, password)

    def get_headers(self):
        return {
            'Authorization': 'Basic %s' % self.authorization,
            'Content-Type': 'application/json'
        }

    def get_my_permissions(self):
        url = JIRA_API_URL + 'mypermissions'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            data = json.loads(result)
            permissions = data['permissions']
        except (JSONDecodeError, KeyError, TypeError):
            raise JiraError
        else:
            return permissions

    def get_projects(self):
        url = JIRA_API_URL + 'project'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            return json.loads(result)
        except JSONDecodeError:
            raise JiraError

    def get_project_versions(self, project_key):
        url = JIRA_API_URL + 'project/' + project_key + '/versions'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            return json.loads(result)
        except JSONDecodeError:
            raise JiraError

    def get_issues(self, jql):
        url = JIRA_API_URL + 'search'
        query = urlencode({
            'jql': jql,
            'maxResults': 100000
        })

        result = make_http_request(url + '?' + query, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            data = json.loads(result)
            issues = data['issues']
        except (JSONDecodeError, KeyError):
            raise JiraError
        else:
            return issues

    def get_issue(self, key):
        url = JIRA_API_URL + 'issue/' + key

        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            issue = json.loads(result)
        except JSONDecodeError:
            raise JiraError
        else:
            return issue

class JiraError(Exception):
    pass

class JiraConnectionError(JiraError):
    pass
<|endoftext|>


<commit_before># std
import os
import logging
import glob
import signal
import subprocess
import tempfile
import time

# datadog
from util import PidFile, yaml, yLoader, get_os

log = logging.getLogger(__name__)

JAVA_LOGGING_LEVEL = {
    logging.CRITICAL : "FATAL",
    logging.DEBUG : "DEBUG",
    logging.ERROR : "ERROR",
    logging.FATAL : "FATAL",
    logging.INFO : "INFO",
    logging.WARN : "WARN",
    logging.WARNING : "WARN",
}

JMX_CHECKS = ['tomcat', 'activemq', 'activemq_58', 'solr', 'cassandra', 'jmx']
JMX_FETCH_JAR_NAME = "jmxfetch-0.2.0-jar-with-dependencies.jar"
JMX_COLLECT_COMMAND = 'collect'
JMX_LIST_COMMANDS = {
        'list_everything': 'List every attributes available that has a type supported by JMXFetch',
        'list_collected_attributes': 'List attributes that will actually be collected by your current instances configuration',
        'list_matching_attributes': 'List attributes that match at least one of your instances configuration',
        'list_not_matching_attributes': "List attributes that don't match any of your instances configuration",
        'list_limited_attributes': "List attributes that do match one of your instances configuration but that are not being collected because it would exceed the number of metrics that can be collected",
        JMX_COLLECT_COMMAND: "Start the collection of metrics based on your current configuration and display them in the console"
        }

PYTHON_JMX_STATUS_FILE = 'jmx_status_python.yaml'

LINK_TO_DOC = "See http://docs.datadoghq.com/integrations/java/ for more information"

class InvalidJMXConfiguration(Exception): pass

class JMXFetch(object):

    pid_file = PidFile("jmxfetch")
    pid_file_path = pid_file.get_path()

    @classmethod
    def init(cls, confd_path, agentConfig, logging_config,
        default_check_frequency, command=None, checks_list=None, reporter=None):
        try:
            jmx_checks, invalid_checks, java_bin_path, java_options = JMXFetch.should_run(confd_path, checks_list)
            try:
                JMXFetch.write_status_file(invalid_checks)
            except Exception:
                log.exception("Error while writing JMX status file")

            if len(jmx_checks) > 0:
                if JMXFetch.is_running() and command == JMX_COLLECT_COMMAND:
                    log.warning("JMXFetch is already running, restarting it.")
                    JMXFetch.stop()

                JMXFetch.start(confd_path, agentConfig, logging_config,
                    java_bin_path, java_options, default_check_frequency,
                    jmx_checks, command, reporter)
                return True
            else:
                return False
        except Exception, e:
            log.exception("Error while initiating JMXFetch")

    @classmethod
    def write_status_file(cls, invalid_checks):
        data = {
            'timestamp':  time.time(),
            'invalid_checks': invalid_checks
        }
        stream = file(os.path.join(tempfile.gettempdir(), PYTHON_JMX_STATUS_FILE), 'w')
        yaml.dump(data, stream)
        stream.close()

    @classmethod
    def should_run(cls, confd_path, checks_list):
        """
    Return a tuple (jmx_checks, invalid_checks, java_bin_path, java_options)

    jmx_checks: list of yaml files that are jmx checks
    (they have the is_jmx flag enabled or they are in JMX_CHECKS)
    and that have at least one instance configured

    invalid_checks: dictionary whose keys are check names that are JMX checks but
    they have a bad configuration. Values of the dictionary are exceptions generated
    when checking the configuration

    java_bin_path: is the path to the java executable. It was
    previously set in the "instance" part of the yaml file of the
    jmx check. So we need to parse yaml files to get it.
    We assume that this value is alwayws the same for every jmx check
    so we can return the first value returned

    java_options: is string contains options that will be passed to java_bin_path
    We assume that this value is alwayws the same for every jmx check
    so we can return the first value returned
    """

        jmx_checks = []
        java_bin_path = None
        java_options = None
        invalid_checks = {}

        for conf in glob.glob(os.path.join(confd_path, '*.yaml')):

            java_bin_path_is_set = java_bin_path is not None
            java_options_is_set = java_options is not None
            filename = os.path.basename(conf)
            check_name = filename.split('.')[0]

            if os.path.exists(conf):
                f = open(conf)
                try:
                    check_config = yaml.load(f.read(), Loader=yLoader)
                    assert check_config is not None
                    f.close()
                except Exception:
                    f.close()
                    log.error("Unable to parse yaml config in %s" % conf)
                    continue

                try:
                    is_jmx, check_java_bin_path, check_java_options = JMXFetch.is_jmx_check(check_config, check_name, checks_list)
                    if is_jmx:
                        jmx_checks.append(filename)
                        if java_bin_path is None and check_java_bin_path is not None:
                            java_bin_path = check_java_bin_path
                        if java_options is None and check_java_options is not None:
                            java_options = check_java_options
                except InvalidJMXConfiguration, e:
                    log.error("%s check is not a valid jmx configuration: %s" % (check_name, e))
                    invalid_checks[check_name] = e

        return (jmx_checks, invalid_checks, java_bin_path, java_options)

    @classmethod
    def is_jmx_check(cls, check_config, check_name, checks_list):
        init_config = check_config.get('init_config', {})
        java_bin_path = None
        java_options = None
        is_jmx = False
        if init_config is None:
            init_config = {}

        if checks_list:
            if check_name in checks_list:
                is_jmx = True

        elif init_config.get('is_jmx') or check_name in JMX_CHECKS:
            is_jmx = True

        if is_jmx:
            instances = check_config.get('instances', [])
            if type(instances) != list or len(instances) == 0:
                raise InvalidJMXConfiguration('You need to have at least one instance defined in the YAML file for this check')

            for inst in instances:
                if type(inst) != dict:
                    raise InvalidJMXConfiguration("Each instance should be a dictionary. %s" % LINK_TO_DOC)
                host = inst.get('host', None)
                port = inst.get('port', None)
                conf = inst.get('conf', init_config.get('conf', None))
                if host is None:
                    raise InvalidJMXConfiguration("A host must be specified")
                if port is None or type(port) != int:
                    raise InvalidJMXConfiguration("A numeric port must be specified")

                if conf is None:
                    log.warning("%s doesn't have a 'conf' section. Only basic JVM metrics will be collected. %s" % (inst, LINK_TO_DOC))
                else:
                    if type(conf) != list or len(conf) == 0:
                        raise InvalidJMXConfiguration("'conf' section should be a list of configurations %s" % LINK_TO_DOC)

                    for config in conf:
                        include = config.get('include', None)
                        if include is None:
                            raise InvalidJMXConfiguration("Each configuration must have an 'include' section. %s" % LINK_TO_DOC)

                        if type(include) != dict:
                            raise InvalidJMXConfiguration("'include' section must be a dictionary %s" % LINK_TO_DOC)

            if java_bin_path is None:
                if init_config and init_config.get('java_bin_path'):
                    # We get the java bin path from the yaml file for backward compatibility purposes
                    java_bin_path = init_config.get('java_bin_path')

                else:
                    for instance in instances:
                        if instance and instance.get('java_bin_path'):
                            java_bin_path = instance.get('java_bin_path')

            if java_options is None:
                if init_config and init_config.get('java_options'):
                    java_options = init_config.get('java_options')
                else:
                    for instance in instances:
                        if instance and instance.get('java_options'):
                            java_options = instance.get('java_options')

        return is_jmx, java_bin_path, java_options

    @classmethod
    def is_running(cls):
        try:
            pid = JMXFetch.pid_file.get_pid()
            if pid is None:
                return False
        except Exception:
            return False

        if get_os() != 'windows':
            try:
                os.kill(pid, 0)
                # os.kill(pid, 0) will throw an exception if pid is not running
                # and won't do anything otherwise
                # It doesn't work on windows as signal.CTRL_C_EVENT is 0, it would quit the process
                return True
            except Exception, e:
                if "Errno 3" not in str(e):
                    log.debug("Couldn't determine if JMXFetch is running. We suppose it's not. %s" % str(e))
                return False

        # Else we are on windows, we need another way to check if it's running
        try:
            import ctypes # Available from python2.5
            kernel32 = ctypes.windll.kernel32
            SYNCHRONIZE = 0x100000

            process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid)
            if process != 0:
                kernel32.CloseHandle(process)
                return True
            else:
                return False

        except Exception, e:
            log.debug("Couldn't determine if JMXFetch is running. We suppose it's not. %s" % str(e))
            return False

    @classmethod
    def stop(cls):
        try:
            pid = JMXFetch.pid_file.get_pid()
            if pid is None:
                log.error("Couldn't get jmxfetch pid.")
                return
        except Exception:
            log.error("Couldn't get jmxfetch pid.")
            return

        try:
            log.info("Killing JMX Fetch")
            os.kill(pid, signal.SIGTERM)
            JMXFetch.pid_file.clean()
            try:
                os.remove(os.path.join(tempfile.gettempdir(), PYTHON_JMX_STATUS_FILE))
            except Exception:
                pass
            log.info("Success")
        except Exception:
            log.error("Couldn't kill jmxfetch pid %s" % pid)

    @classmethod
    def get_path_to_jmxfetch(cls):
        if get_os() != 'windows':
            return os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "checks", "libs", JMX_FETCH_JAR_NAME))

        return os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "..", "jmxfetch", JMX_FETCH_JAR_NAME))

    @classmethod
    def start(cls, confd_path, agentConfig, logging_config, path_to_java, java_run_opts,
        default_check_frequency, jmx_checks, command=None, reporter=None):
        statsd_port = agentConfig.get('dogstatsd_port', "8125")

        command = command or JMX_COLLECT_COMMAND
        if reporter is None:
            reporter = "statsd:%s" % str(statsd_port)

        log.info("Starting jmxfetch:")
        jmx_connector_pid = None
        try:
            path_to_java = path_to_java or "java"
            java_run_opts = java_run_opts or ""
            path_to_jmxfetch = JMXFetch.get_path_to_jmxfetch()
            path_to_status_file = os.path.join(tempfile.gettempdir(), "jmx_status.yaml")

            subprocess_args = [
                path_to_java, # Path to the java bin
                '-jar',
                r"%s" % path_to_jmxfetch, # Path to the jmxfetch jar
                '--check_period', str(default_check_frequency * 1000),  # Period of the main loop of jmxfetch in ms
                '--conf_directory', r"%s" % confd_path, # Path of the conf.d directory that will be read by jmxfetch,
                '--log_level', JAVA_LOGGING_LEVEL.get(logging_config.get("log_level"), "INFO"),  # Log Level: Mapping from Python log level to log4j log levels
                '--log_location', r"%s" % logging_config.get('jmxfetch_log_file'), # Path of the log file
                '--reporter',  reporter, # Reporter to use
                '--status_location', r"%s" % path_to_status_file, # Path to the status file to write
                command, # Name of the command
            ]

            subprocess_args.insert(3, '--check')
            for check in jmx_checks:
                subprocess_args.insert(4, check)

            if java_run_opts:
                for opt in java_run_opts.split():
                    subprocess_args.insert(1,opt)

            log.info("Running %s" % " ".join(subprocess_args))
            if reporter != "console":
                cls.subprocess = subprocess.Popen(subprocess_args, close_fds=True)
                jmx_connector_pid = cls.subprocess.pid
                log.debug("JMX Fetch pid: %s" % jmx_connector_pid)

            else:
                subprocess.call(subprocess_args)

        except OSError, e:
            jmx_connector_pid = None
            log.exception("Couldn't launch JMXFetch. Is java in your PATH?")
        except Exception, e:
            jmx_connector_pid = None
            log.exception("Couldn't launch JMXFetch")

        # Write pid to pid file
        if jmx_connector_pid is not None:
            try:
                fp = open(JMXFetch.pid_file_path, 'w+')
                fp.write(str(jmx_connector_pid))
                fp.close()
                os.chmod(JMXFetch.pid_file_path, 0644)
            except Exception, e:
                log.exception("Unable to write JMXFetch pidfile: %s" % JMXFetch.pid_file_path)

<commit_msg>Only write jmx python status file if there are invalid jmx checks<commit_after># std
import os
import logging
import glob
import signal
import subprocess
import tempfile
import time

# datadog
from util import PidFile, yaml, yLoader, get_os

log = logging.getLogger(__name__)

JAVA_LOGGING_LEVEL = {
    logging.CRITICAL : "FATAL",
    logging.DEBUG : "DEBUG",
    logging.ERROR : "ERROR",
    logging.FATAL : "FATAL",
    logging.INFO : "INFO",
    logging.WARN : "WARN",
    logging.WARNING : "WARN",
}

JMX_CHECKS = ['tomcat', 'activemq', 'activemq_58', 'solr', 'cassandra', 'jmx']
JMX_FETCH_JAR_NAME = "jmxfetch-0.2.0-jar-with-dependencies.jar"
JMX_COLLECT_COMMAND = 'collect'
JMX_LIST_COMMANDS = {
        'list_everything': 'List every attributes available that has a type supported by JMXFetch',
        'list_collected_attributes': 'List attributes that will actually be collected by your current instances configuration',
        'list_matching_attributes': 'List attributes that match at least one of your instances configuration',
        'list_not_matching_attributes': "List attributes that don't match any of your instances configuration",
        'list_limited_attributes': "List attributes that do match one of your instances configuration but that are not being collected because it would exceed the number of metrics that can be collected",
        JMX_COLLECT_COMMAND: "Start the collection of metrics based on your current configuration and display them in the console"
        }

PYTHON_JMX_STATUS_FILE = 'jmx_status_python.yaml'

LINK_TO_DOC = "See http://docs.datadoghq.com/integrations/java/ for more information"

class InvalidJMXConfiguration(Exception): pass

class JMXFetch(object):

    pid_file = PidFile("jmxfetch")
    pid_file_path = pid_file.get_path()

    @classmethod
    def init(cls, confd_path, agentConfig, logging_config,
        default_check_frequency, command=None, checks_list=None, reporter=None):
        try:
            jmx_checks, invalid_checks, java_bin_path, java_options = JMXFetch.should_run(confd_path, checks_list)
            if len(invalid_checks) > 0:
                try:
                    JMXFetch.write_status_file(invalid_checks)
                except Exception:
                    log.exception("Error while writing JMX status file")

            if len(jmx_checks) > 0:
                if JMXFetch.is_running() and command == JMX_COLLECT_COMMAND:
                    log.warning("JMXFetch is already running, restarting it.")
                    JMXFetch.stop()

                JMXFetch.start(confd_path, agentConfig, logging_config,
                    java_bin_path, java_options, default_check_frequency,
                    jmx_checks, command, reporter)
                return True
            else:
                return False
        except Exception, e:
            log.exception("Error while initiating JMXFetch")

    @classmethod
    def write_status_file(cls, invalid_checks):
        data = {
            'timestamp':  time.time(),
            'invalid_checks': invalid_checks
        }
        stream = file(os.path.join(tempfile.gettempdir(), PYTHON_JMX_STATUS_FILE), 'w')
        yaml.dump(data, stream)
        stream.close()

    @classmethod
    def should_run(cls, confd_path, checks_list):
        """
    Return a tuple (jmx_checks, invalid_checks, java_bin_path, java_options)

    jmx_checks: list of yaml files that are jmx checks
    (they have the is_jmx flag enabled or they are in JMX_CHECKS)
    and that have at least one instance configured

    invalid_checks: dictionary whose keys are check names that are JMX checks but
    they have a bad configuration. Values of the dictionary are exceptions generated
    when checking the configuration

    java_bin_path: is the path to the java executable. It was
    previously set in the "instance" part of the yaml file of the
    jmx check. So we need to parse yaml files to get it.
    We assume that this value is alwayws the same for every jmx check
    so we can return the first value returned

    java_options: is string contains options that will be passed to java_bin_path
    We assume that this value is alwayws the same for every jmx check
    so we can return the first value returned
    """

        jmx_checks = []
        java_bin_path = None
        java_options = None
        invalid_checks = {}

        for conf in glob.glob(os.path.join(confd_path, '*.yaml')):

            java_bin_path_is_set = java_bin_path is not None
            java_options_is_set = java_options is not None
            filename = os.path.basename(conf)
            check_name = filename.split('.')[0]

            if os.path.exists(conf):
                f = open(conf)
                try:
                    check_config = yaml.load(f.read(), Loader=yLoader)
                    assert check_config is not None
                    f.close()
                except Exception:
                    f.close()
                    log.error("Unable to parse yaml config in %s" % conf)
                    continue

                try:
                    is_jmx, check_java_bin_path, check_java_options = JMXFetch.is_jmx_check(check_config, check_name, checks_list)
                    if is_jmx:
                        jmx_checks.append(filename)
                        if java_bin_path is None and check_java_bin_path is not None:
                            java_bin_path = check_java_bin_path
                        if java_options is None and check_java_options is not None:
                            java_options = check_java_options
                except InvalidJMXConfiguration, e:
                    log.error("%s check is not a valid jmx configuration: %s" % (check_name, e))
                    invalid_checks[check_name] = e

        return (jmx_checks, invalid_checks, java_bin_path, java_options)

    @classmethod
    def is_jmx_check(cls, check_config, check_name, checks_list):
        init_config = check_config.get('init_config', {})
        java_bin_path = None
        java_options = None
        is_jmx = False
        if init_config is None:
            init_config = {}

        if checks_list:
            if check_name in checks_list:
                is_jmx = True

        elif init_config.get('is_jmx') or check_name in JMX_CHECKS:
            is_jmx = True

        if is_jmx:
            instances = check_config.get('instances', [])
            if type(instances) != list or len(instances) == 0:
                raise InvalidJMXConfiguration('You need to have at least one instance defined in the YAML file for this check')

            for inst in instances:
                if type(inst) != dict:
                    raise InvalidJMXConfiguration("Each instance should be a dictionary. %s" % LINK_TO_DOC)
                host = inst.get('host', None)
                port = inst.get('port', None)
                conf = inst.get('conf', init_config.get('conf', None))
                if host is None:
                    raise InvalidJMXConfiguration("A host must be specified")
                if port is None or type(port) != int:
                    raise InvalidJMXConfiguration("A numeric port must be specified")

                if conf is None:
                    log.warning("%s doesn't have a 'conf' section. Only basic JVM metrics will be collected. %s" % (inst, LINK_TO_DOC))
                else:
                    if type(conf) != list or len(conf) == 0:
                        raise InvalidJMXConfiguration("'conf' section should be a list of configurations %s" % LINK_TO_DOC)

                    for config in conf:
                        include = config.get('include', None)
                        if include is None:
                            raise InvalidJMXConfiguration("Each configuration must have an 'include' section. %s" % LINK_TO_DOC)

                        if type(include) != dict:
                            raise InvalidJMXConfiguration("'include' section must be a dictionary %s" % LINK_TO_DOC)

            if java_bin_path is None:
                if init_config and init_config.get('java_bin_path'):
                    # We get the java bin path from the yaml file for backward compatibility purposes
                    java_bin_path = init_config.get('java_bin_path')

                else:
                    for instance in instances:
                        if instance and instance.get('java_bin_path'):
                            java_bin_path = instance.get('java_bin_path')

            if java_options is None:
                if init_config and init_config.get('java_options'):
                    java_options = init_config.get('java_options')
                else:
                    for instance in instances:
                        if instance and instance.get('java_options'):
                            java_options = instance.get('java_options')

        return is_jmx, java_bin_path, java_options

    @classmethod
    def is_running(cls):
        try:
            pid = JMXFetch.pid_file.get_pid()
            if pid is None:
                return False
        except Exception:
            return False

        if get_os() != 'windows':
            try:
                os.kill(pid, 0)
                # os.kill(pid, 0) will throw an exception if pid is not running
                # and won't do anything otherwise
                # It doesn't work on windows as signal.CTRL_C_EVENT is 0, it would quit the process
                return True
            except Exception, e:
                if "Errno 3" not in str(e):
                    log.debug("Couldn't determine if JMXFetch is running. We suppose it's not. %s" % str(e))
                return False

        # Else we are on windows, we need another way to check if it's running
        try:
            import ctypes # Available from python2.5
            kernel32 = ctypes.windll.kernel32
            SYNCHRONIZE = 0x100000

            process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid)
            if process != 0:
                kernel32.CloseHandle(process)
                return True
            else:
                return False

        except Exception, e:
            log.debug("Couldn't determine if JMXFetch is running. We suppose it's not. %s" % str(e))
            return False

    @classmethod
    def stop(cls):
        try:
            pid = JMXFetch.pid_file.get_pid()
            if pid is None:
                log.error("Couldn't get jmxfetch pid.")
                return
        except Exception:
            log.error("Couldn't get jmxfetch pid.")
            return

        try:
            log.info("Killing JMX Fetch")
            os.kill(pid, signal.SIGTERM)
            JMXFetch.pid_file.clean()
            try:
                os.remove(os.path.join(tempfile.gettempdir(), PYTHON_JMX_STATUS_FILE))
            except Exception:
                pass
            log.info("Success")
        except Exception:
            log.error("Couldn't kill jmxfetch pid %s" % pid)

    @classmethod
    def get_path_to_jmxfetch(cls):
        if get_os() != 'windows':
            return os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "checks", "libs", JMX_FETCH_JAR_NAME))

        return os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "..", "jmxfetch", JMX_FETCH_JAR_NAME))

    @classmethod
    def start(cls, confd_path, agentConfig, logging_config, path_to_java, java_run_opts,
        default_check_frequency, jmx_checks, command=None, reporter=None):
        statsd_port = agentConfig.get('dogstatsd_port', "8125")

        command = command or JMX_COLLECT_COMMAND
        if reporter is None:
            reporter = "statsd:%s" % str(statsd_port)

        log.info("Starting jmxfetch:")
        jmx_connector_pid = None
        try:
            path_to_java = path_to_java or "java"
            java_run_opts = java_run_opts or ""
            path_to_jmxfetch = JMXFetch.get_path_to_jmxfetch()
            path_to_status_file = os.path.join(tempfile.gettempdir(), "jmx_status.yaml")

            subprocess_args = [
                path_to_java, # Path to the java bin
                '-jar',
                r"%s" % path_to_jmxfetch, # Path to the jmxfetch jar
                '--check_period', str(default_check_frequency * 1000),  # Period of the main loop of jmxfetch in ms
                '--conf_directory', r"%s" % confd_path, # Path of the conf.d directory that will be read by jmxfetch,
                '--log_level', JAVA_LOGGING_LEVEL.get(logging_config.get("log_level"), "INFO"),  # Log Level: Mapping from Python log level to log4j log levels
                '--log_location', r"%s" % logging_config.get('jmxfetch_log_file'), # Path of the log file
                '--reporter',  reporter, # Reporter to use
                '--status_location', r"%s" % path_to_status_file, # Path to the status file to write
                command, # Name of the command
            ]

            subprocess_args.insert(3, '--check')
            for check in jmx_checks:
                subprocess_args.insert(4, check)

            if java_run_opts:
                for opt in java_run_opts.split():
                    subprocess_args.insert(1,opt)

            log.info("Running %s" % " ".join(subprocess_args))
            if reporter != "console":
                cls.subprocess = subprocess.Popen(subprocess_args, close_fds=True)
                jmx_connector_pid = cls.subprocess.pid
                log.debug("JMX Fetch pid: %s" % jmx_connector_pid)

            else:
                subprocess.call(subprocess_args)

        except OSError, e:
            jmx_connector_pid = None
            log.exception("Couldn't launch JMXFetch. Is java in your PATH?")
        except Exception, e:
            jmx_connector_pid = None
            log.exception("Couldn't launch JMXFetch")

        # Write pid to pid file
        if jmx_connector_pid is not None:
            try:
                fp = open(JMXFetch.pid_file_path, 'w+')
                fp.write(str(jmx_connector_pid))
                fp.close()
                os.chmod(JMXFetch.pid_file_path, 0644)
            except Exception, e:
                log.exception("Unable to write JMXFetch pidfile: %s" % JMXFetch.pid_file_path)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm, fields
from openerp.tools.translate import _

class picking_dispatch_apply_carrier(orm.TransientModel):
    _name = 'picking.dispatch.apply.carrier'
    _description = 'Picking Dispatch Apply Carrier'

    _columns = {
        'carrier_id': fields.many2one(
            'delivery.carrier',
            string='Carrier',
            required=True),
    }

    def _check_domain(self, cr, uid, ids, dispatch_ids, context=None):
        """ A domain excluding the dispatches on which we don't allow
        to change the carrier.

        Can be overrided to change the domain.
        """
        return [('state', '!=', 'done'),
                ('id', 'in', dispatch_ids)]

    def apply(self, cr, uid, ids, context=None):
        if context is None:
            context = {}

        dispatch_ids = context.get('active_ids')
        if not dispatch_ids:
            raise orm.except_orm(
                _('Error'),
                _('No selected picking dispatch'))

        assert len(ids) == 1
        this = self.browse(cr, uid, ids[0], context=context)

        dispatch_obj = self.pool['picking.dispatch']
        domain = self._check_domain(cr, uid, ids, dispatch_ids, context=context)
        dispatch_ids = dispatch_obj.search(cr, uid, domain, context=context)
        dispatch_obj.write(cr, uid, dispatch_ids,
                           {'carrier_id': this.carrier_id.id},
                           context=context)
        dispatch_obj.action_set_options(cr, uid, dispatch_ids, context=context)
        return {'type': 'ir.actions.act_window_close'}
<commit_msg>Too long line<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm, fields
from openerp.tools.translate import _

class picking_dispatch_apply_carrier(orm.TransientModel):
    _name = 'picking.dispatch.apply.carrier'
    _description = 'Picking Dispatch Apply Carrier'

    _columns = {
        'carrier_id': fields.many2one(
            'delivery.carrier',
            string='Carrier',
            required=True),
    }

    def _check_domain(self, cr, uid, ids, dispatch_ids, context=None):
        """ A domain excluding the dispatches on which we don't allow
        to change the carrier.

        Can be overrided to change the domain.
        """
        return [('state', '!=', 'done'),
                ('id', 'in', dispatch_ids)]

    def apply(self, cr, uid, ids, context=None):
        if context is None:
            context = {}

        dispatch_ids = context.get('active_ids')
        if not dispatch_ids:
            raise orm.except_orm(
                _('Error'),
                _('No selected picking dispatch'))

        assert len(ids) == 1
        this = self.browse(cr, uid, ids[0], context=context)

        dispatch_obj = self.pool['picking.dispatch']
        domain = self._check_domain(cr, uid, ids, dispatch_ids,
                                    context=context)
        dispatch_ids = dispatch_obj.search(cr, uid, domain, context=context)
        dispatch_obj.write(cr, uid, dispatch_ids,
                           {'carrier_id': this.carrier_id.id},
                           context=context)
        dispatch_obj.action_set_options(cr, uid, dispatch_ids, context=context)
        return {'type': 'ir.actions.act_window_close'}
<|endoftext|>


<commit_before>"""
    Script to files (mainly images) from one dir to another

    Reads list of files to move. If images does not verify whether they are
    corrupt - simply moves from one directory to that other creating the
    required directory structure if necessary
"""

import os
import argparse

parser = argparse.ArgumentParser(
            description="Move files (images) from one dir to another")

parser.add_argument('-i', dest='inputFiles', required=True,
                    help='File containing list of files/images to move'
)
parser.add_argument('-s', dest='splitString',
                    help='token to separate the basedir from input files'
)
parser.add_argument('-r', dest='replacementString',
                    help='String to replace the split string with'
)
parser.add_argument('-d', dest='destDirBase', required=True,
                    help='Path to the base of the destination dir'
)

args = parser.parse_args()

input_files = args.inputFiles
split_string = "" if args.splitString is None else args.splitString
replacement_string = "" if args.replacementString is None else args.replacementString

n_success = 0
n_failed = 0
with open(input_files,'rt') as f:
    fnames = [fname.strip('\n') for fname in f.readlines()]
    n_total = len(fnames)
    if n_total == 0:
        print(f"No files to move found in {input_files}")
    for fname in fnames:
        try:
            fname2 = fname.replace(split_string, replacement_string)
            os.renames(fname, fname2)
            n_success += 1
        except Exception as e:
            str_err = str(e)
            print(f"Failed to move {fname2} error was: {str_err}")
            n_failed += 1

if n_success > 0:
    print(f"Successfully moved {n_success}/{n_total} files")
if n_failed > 0:
    print(f"Problems moving {n_failed}/{n_total} files")
<commit_msg>Remove unnecessary command line parameter<commit_after>"""
    Script to files (mainly images) from one dir to another

    Reads list of files to move. If images does not verify whether they are
    corrupt - simply moves from one directory to that other creating the
    required directory structure if necessary
"""

import os
import argparse

parser = argparse.ArgumentParser(
            description="Move files (images) from one dir to another")

parser.add_argument('-i', dest='inputFiles', required=True,
                    help='File containing list of files/images to move'
)
parser.add_argument('-s', dest='splitString',
                    help='token to separate the basedir from input files'
)
parser.add_argument('-r', dest='replacementString',
                    help='String to replace the split string with'
)

args = parser.parse_args()

input_files = args.inputFiles
split_string = "" if args.splitString is None else args.splitString
replacement_string = "" if args.replacementString is None else args.replacementString

n_success = 0
n_failed = 0
with open(input_files,'rt') as f:
    fnames = [fname.strip('\n') for fname in f.readlines()]
    n_total = len(fnames)
    if n_total == 0:
        print(f"No files to move found in {input_files}")
    for fname in fnames:
        try:
            fname2 = fname.replace(split_string, replacement_string)
            os.renames(fname, fname2)
            n_success += 1
        except Exception as e:
            str_err = str(e)
            print(f"Failed to move {fname2} error was: {str_err}")
            n_failed += 1

if n_success > 0:
    print(f"Successfully moved {n_success}/{n_total} files")
if n_failed > 0:
    print(f"Problems moving {n_failed}/{n_total} files")
<|endoftext|>


<commit_before>import os
import pytest
import cPickle as pickle
from modules.beautify.beautify import beautify, json_return, has_failed
from tests.constants import ARGS_DICT, BEAUTIFY_VF_SEROTYPE

vf_serotype_gene_dict = os.path.join('tests/refs', 'GCA_000005845.2_ASM584v2_genomic.fna_ectyper-vf_serotype.p')

amr_gene_dict = os.path.join('tests/refs', '2017-05-21-00-29-20-874628-GCA_000005845.2_ASM584v2_genomic.fna_rgi.tsv_rgi.p')

def test_beautify_vf_serotype():
    ## test vf & serotype json return
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': vf_serotype_gene_dict})
    assert len(beautify(single_dict, vf_serotype_gene_dict)) == len(BEAUTIFY_VF_SEROTYPE)

def test_beautify_serotype_only():
    ## test serotype only json return
    # note: this is actually the same gene results file as above
    # we only differentiate what is returned to the user, because we want all analysis ran & added to the db
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': vf_serotype_gene_dict})
    # this mimicks user selection of serotype only
    single_dict.update({'options':{'vf': False, 'amr': False, 'serotype': True}})
    # beautify is what is actually called by the RQ worker & returned to the user
    r = beautify(single_dict, vf_serotype_gene_dict)
    print r
    assert len(r) == 1

def test_beautify_json_r_serotype_only():
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': vf_serotype_gene_dict})
    # this mimicks user selection of serotype only
    single_dict.update({'options':{'vf': False, 'amr': False, 'serotype': True}})
    ## test json_r separately of failed handling
    # json_return() is a part of the beautify work
    gene_dict = pickle.load(open(vf_serotype_gene_dict, 'rb'))
    print type(gene_dict)
    r = json_return(single_dict, gene_dict)
    assert len(r) == 1

    failed = has_failed(r)
    assert failed == False

def test_beautify_amr_only():
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': amr_gene_dict})
    # this mimicks user selection of serotype only
    single_dict.update({'options':{'vf': False, 'amr': True, 'serotype': False}})
    r = beautify(single_dict, amr_gene_dict)
    assert len(r) > 1
<commit_msg>ADD: check type of pickled files<commit_after>import os
import pytest
import cPickle as pickle
from modules.beautify.beautify import beautify, json_return, has_failed
from tests.constants import ARGS_DICT, BEAUTIFY_VF_SEROTYPE

vf_serotype_gene_dict = os.path.join('tests/refs', 'GCA_000005845.2_ASM584v2_genomic.fna_ectyper-vf_serotype.p')

amr_gene_dict = os.path.join('tests/refs', '2017-05-21-00-29-20-874628-GCA_000005845.2_ASM584v2_genomic.fna_rgi.tsv_rgi.p')

def test_beautify_vf_serotype():
    ## test vf & serotype json return
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': vf_serotype_gene_dict})
    assert len(beautify(single_dict, vf_serotype_gene_dict)) == len(BEAUTIFY_VF_SEROTYPE)

def test_beautify_serotype_only():
    ## test serotype only json return
    # note: this is actually the same gene results file as above
    # we only differentiate what is returned to the user, because we want all analysis ran & added to the db
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': vf_serotype_gene_dict})
    # this mimicks user selection of serotype only
    single_dict.update({'options':{'vf': False, 'amr': False, 'serotype': True}})
    # beautify is what is actually called by the RQ worker & returned to the user
    r = beautify(single_dict, vf_serotype_gene_dict)
    print r
    assert len(r) == 1

def test_beautify_json_r_serotype_only():
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': vf_serotype_gene_dict})
    # this mimicks user selection of serotype only
    single_dict.update({'options':{'vf': False, 'amr': False, 'serotype': True}})
    ## test json_r separately of failed handling
    # json_return() is a part of the beautify work
    gene_dict = pickle.load(open(vf_serotype_gene_dict, 'rb'))
    assert type(gene_dict) == list
    r = json_return(single_dict, gene_dict)
    assert len(r) == 1

    failed = has_failed(r)
    assert failed == False

def test_beautify_amr_only():
    single_dict = dict(ARGS_DICT)
    single_dict.update({'i': amr_gene_dict})
    # this mimicks user selection of serotype only
    single_dict.update({'options':{'vf': False, 'amr': True, 'serotype': False}})
    r = beautify(single_dict, amr_gene_dict)
    assert len(r) > 1
<|endoftext|>


<commit_before>import re
import sys

from django.contrib.gis.db.backends.base import BaseSpatialOperations
from django.contrib.gis.db.backends.utils import SpatialOperator
from django.contrib.gis.db.backends.spatialite.adapter import SpatiaLiteAdapter
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Distance
from django.core.exceptions import ImproperlyConfigured
from django.db.backends.sqlite3.base import DatabaseOperations
from django.db.utils import DatabaseError
from django.utils import six
from django.utils.functional import cached_property

class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations):
    compiler_module = 'django.contrib.gis.db.models.sql.compiler'
    name = 'spatialite'
    spatialite = True
    version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)')

    @property
    def valid_aggregates(self):
        if self.spatial_version >= 3:
            return {'Collect', 'Extent', 'Union'}
        else:
            return {'Union'}

    Adapter = SpatiaLiteAdapter
    Adaptor = Adapter  # Backwards-compatibility alias.

    area = 'Area'
    centroid = 'Centroid'
    collect = 'Collect'
    contained = 'MbrWithin'
    difference = 'Difference'
    distance = 'Distance'
    envelope = 'Envelope'
    extent = 'Extent'
    intersection = 'Intersection'
    length = 'GLength'  # OpenGis defines Length, but this conflicts with an SQLite reserved keyword
    num_geom = 'NumGeometries'
    num_points = 'NumPoints'
    point_on_surface = 'PointOnSurface'
    scale = 'ScaleCoords'
    svg = 'AsSVG'
    sym_difference = 'SymDifference'
    transform = 'Transform'
    translate = 'ShiftCoords'
    union = 'GUnion'  # OpenGis defines Union, but this conflicts with an SQLite reserved keyword
    unionagg = 'GUnion'

    from_text = 'GeomFromText'
    from_wkb = 'GeomFromWKB'
    select = 'AsText(%s)'

    gis_operators = {
        'equals': SpatialOperator(func='Equals'),
        'disjoint': SpatialOperator(func='Disjoint'),
        'touches': SpatialOperator(func='Touches'),
        'crosses': SpatialOperator(func='Crosses'),
        'within': SpatialOperator(func='Within'),
        'overlaps': SpatialOperator(func='Overlaps'),
        'contains': SpatialOperator(func='Contains'),
        'intersects': SpatialOperator(func='Intersects'),
        'relate': SpatialOperator(func='Relate'),
        # Returns true if B's bounding box completely contains A's bounding box.
        'contained': SpatialOperator(func='MbrWithin'),
        # Returns true if A's bounding box completely contains B's bounding box.
        'bbcontains': SpatialOperator(func='MbrContains'),
        # Returns true if A's bounding box overlaps B's bounding box.
        'bboverlaps': SpatialOperator(func='MbrOverlaps'),
        # These are implemented here as synonyms for Equals
        'same_as': SpatialOperator(func='Equals'),
        'exact': SpatialOperator(func='Equals'),

        'distance_gt': SpatialOperator(func='Distance', op='>'),
        'distance_gte': SpatialOperator(func='Distance', op='>='),
        'distance_lt': SpatialOperator(func='Distance', op='<'),
        'distance_lte': SpatialOperator(func='Distance', op='<='),
    }

    @cached_property
    def spatial_version(self):
        """Determine the version of the SpatiaLite library."""
        try:
            version = self.spatialite_version_tuple()[1:]
        except Exception as msg:
            new_msg = (
                'Cannot determine the SpatiaLite version for the "%s" '
                'database (error was "%s").  Was the SpatiaLite initialization '
                'SQL loaded on this database?') % (self.connection.settings_dict['NAME'], msg)
            six.reraise(ImproperlyConfigured, ImproperlyConfigured(new_msg), sys.exc_info()[2])
        if version < (2, 4, 0):
            raise ImproperlyConfigured('GeoDjango only supports SpatiaLite versions '
                                       '2.4.0 and above')
        return version

    @property
    def _version_greater_2_4_0_rc4(self):
        if self.spatial_version >= (2, 4, 1):
            return True
        else:
            # Spatialite 2.4.0-RC4 added AsGML and AsKML, however both
            # RC2 (shipped in popular Debian/Ubuntu packages) and RC4
            # report version as '2.4.0', so we fall back to feature detection
            try:
                self._get_spatialite_func("AsGML(GeomFromText('POINT(1 1)'))")
            except DatabaseError:
                return False
            return True

    @cached_property
    def gml(self):
        return 'AsGML' if self._version_greater_2_4_0_rc4 else None

    @cached_property
    def kml(self):
        return 'AsKML' if self._version_greater_2_4_0_rc4 else None

    @cached_property
    def geojson(self):
        return 'AsGeoJSON' if self.spatial_version >= (3, 0, 0) else None

    def check_aggregate_support(self, aggregate):
        """
        Checks if the given aggregate name is supported (that is, if it's
        in `self.valid_aggregates`).
        """
        super(SpatiaLiteOperations, self).check_aggregate_support(aggregate)
        agg_name = aggregate.__class__.__name__
        return agg_name in self.valid_aggregates

    def convert_extent(self, box):
        """
        Convert the polygon data received from Spatialite to min/max values.
        """
        shell = Geometry(box).shell
        xmin, ymin = shell[0][:2]
        xmax, ymax = shell[2][:2]
        return (xmin, ymin, xmax, ymax)

    def convert_geom(self, wkt, geo_field):
        """
        Converts geometry WKT returned from a SpatiaLite aggregate.
        """
        if wkt:
            return Geometry(wkt, geo_field.srid)
        else:
            return None

    def geo_db_type(self, f):
        """
        Returns None because geometry columnas are added via the
        `AddGeometryColumn` stored procedure on SpatiaLite.
        """
        return None

    def get_distance(self, f, value, lookup_type):
        """
        Returns the distance parameters for the given geometry field,
        lookup value, and lookup type.  SpatiaLite only supports regular
        cartesian-based queries (no spheroid/sphere calculations for point
        geometries like PostGIS).
        """
        if not value:
            return []
        value = value[0]
        if isinstance(value, Distance):
            if f.geodetic(self.connection):
                raise ValueError('SpatiaLite does not support distance queries on '
                                 'geometry fields with a geodetic coordinate system. '
                                 'Distance objects; use a numeric value of your '
                                 'distance in degrees instead.')
            else:
                dist_param = getattr(value, Distance.unit_attname(f.units_name(self.connection)))
        else:
            dist_param = value
        return [dist_param]

    def get_geom_placeholder(self, f, value):
        """
        Provides a proper substitution value for Geometries that are not in the
        SRID of the field.  Specifically, this routine will substitute in the
        Transform() and GeomFromText() function call(s).
        """
        def transform_value(value, srid):
            return not (value is None or value.srid == srid)
        if hasattr(value, 'expression'):
            if transform_value(value, f.srid):
                placeholder = '%s(%%s, %s)' % (self.transform, f.srid)
            else:
                placeholder = '%s'
            # No geometry value used for F expression, substitute in
            # the column name instead.
            return placeholder % self.get_expression_column(value)
        else:
            if transform_value(value, f.srid):
                # Adding Transform() to the SQL placeholder.
                return '%s(%s(%%s,%s), %s)' % (self.transform, self.from_text, value.srid, f.srid)
            else:
                return '%s(%%s,%s)' % (self.from_text, f.srid)

    def _get_spatialite_func(self, func):
        """
        Helper routine for calling SpatiaLite functions and returning
        their result.
        Any error occurring in this method should be handled by the caller.
        """
        cursor = self.connection._cursor()
        try:
            cursor.execute('SELECT %s' % func)
            row = cursor.fetchone()
        finally:
            cursor.close()
        return row[0]

    def geos_version(self):
        "Returns the version of GEOS used by SpatiaLite as a string."
        return self._get_spatialite_func('geos_version()')

    def proj4_version(self):
        "Returns the version of the PROJ.4 library used by SpatiaLite."
        return self._get_spatialite_func('proj4_version()')

    def spatialite_version(self):
        "Returns the SpatiaLite library version as a string."
        return self._get_spatialite_func('spatialite_version()')

    def spatialite_version_tuple(self):
        """
        Returns the SpatiaLite version as a tuple (version string, major,
        minor, subminor).
        """
        version = self.spatialite_version()

        m = self.version_regex.match(version)
        if m:
            major = int(m.group('major'))
            minor1 = int(m.group('minor1'))
            minor2 = int(m.group('minor2'))
        else:
            raise Exception('Could not parse SpatiaLite version string: %s' % version)

        return (version, major, minor1, minor2)

    def spatial_aggregate_sql(self, agg):
        """
        Returns the spatial aggregate SQL template and function for the
        given Aggregate instance.
        """
        agg_name = agg.__class__.__name__
        if not self.check_aggregate_support(agg):
            raise NotImplementedError('%s spatial aggregate is not implemented for this backend.' % agg_name)
        agg_name = agg_name.lower()
        if agg_name == 'union':
            agg_name += 'agg'
        sql_template = self.select % '%(function)s(%(field)s)'
        sql_function = getattr(self, agg_name)
        return sql_template, sql_function

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialiteGeometryColumns
        return SpatialiteGeometryColumns

    def spatial_ref_sys(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialiteSpatialRefSys
        return SpatialiteSpatialRefSys
<commit_msg>Fixed a tuple/int comparison in spatialite backend<commit_after>import re
import sys

from django.contrib.gis.db.backends.base import BaseSpatialOperations
from django.contrib.gis.db.backends.utils import SpatialOperator
from django.contrib.gis.db.backends.spatialite.adapter import SpatiaLiteAdapter
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Distance
from django.core.exceptions import ImproperlyConfigured
from django.db.backends.sqlite3.base import DatabaseOperations
from django.db.utils import DatabaseError
from django.utils import six
from django.utils.functional import cached_property

class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations):
    compiler_module = 'django.contrib.gis.db.models.sql.compiler'
    name = 'spatialite'
    spatialite = True
    version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)')

    @property
    def valid_aggregates(self):
        if self.spatial_version >= (3, 0, 0):
            return {'Collect', 'Extent', 'Union'}
        else:
            return {'Union'}

    Adapter = SpatiaLiteAdapter
    Adaptor = Adapter  # Backwards-compatibility alias.

    area = 'Area'
    centroid = 'Centroid'
    collect = 'Collect'
    contained = 'MbrWithin'
    difference = 'Difference'
    distance = 'Distance'
    envelope = 'Envelope'
    extent = 'Extent'
    intersection = 'Intersection'
    length = 'GLength'  # OpenGis defines Length, but this conflicts with an SQLite reserved keyword
    num_geom = 'NumGeometries'
    num_points = 'NumPoints'
    point_on_surface = 'PointOnSurface'
    scale = 'ScaleCoords'
    svg = 'AsSVG'
    sym_difference = 'SymDifference'
    transform = 'Transform'
    translate = 'ShiftCoords'
    union = 'GUnion'  # OpenGis defines Union, but this conflicts with an SQLite reserved keyword
    unionagg = 'GUnion'

    from_text = 'GeomFromText'
    from_wkb = 'GeomFromWKB'
    select = 'AsText(%s)'

    gis_operators = {
        'equals': SpatialOperator(func='Equals'),
        'disjoint': SpatialOperator(func='Disjoint'),
        'touches': SpatialOperator(func='Touches'),
        'crosses': SpatialOperator(func='Crosses'),
        'within': SpatialOperator(func='Within'),
        'overlaps': SpatialOperator(func='Overlaps'),
        'contains': SpatialOperator(func='Contains'),
        'intersects': SpatialOperator(func='Intersects'),
        'relate': SpatialOperator(func='Relate'),
        # Returns true if B's bounding box completely contains A's bounding box.
        'contained': SpatialOperator(func='MbrWithin'),
        # Returns true if A's bounding box completely contains B's bounding box.
        'bbcontains': SpatialOperator(func='MbrContains'),
        # Returns true if A's bounding box overlaps B's bounding box.
        'bboverlaps': SpatialOperator(func='MbrOverlaps'),
        # These are implemented here as synonyms for Equals
        'same_as': SpatialOperator(func='Equals'),
        'exact': SpatialOperator(func='Equals'),

        'distance_gt': SpatialOperator(func='Distance', op='>'),
        'distance_gte': SpatialOperator(func='Distance', op='>='),
        'distance_lt': SpatialOperator(func='Distance', op='<'),
        'distance_lte': SpatialOperator(func='Distance', op='<='),
    }

    @cached_property
    def spatial_version(self):
        """Determine the version of the SpatiaLite library."""
        try:
            version = self.spatialite_version_tuple()[1:]
        except Exception as msg:
            new_msg = (
                'Cannot determine the SpatiaLite version for the "%s" '
                'database (error was "%s").  Was the SpatiaLite initialization '
                'SQL loaded on this database?') % (self.connection.settings_dict['NAME'], msg)
            six.reraise(ImproperlyConfigured, ImproperlyConfigured(new_msg), sys.exc_info()[2])
        if version < (2, 4, 0):
            raise ImproperlyConfigured('GeoDjango only supports SpatiaLite versions '
                                       '2.4.0 and above')
        return version

    @property
    def _version_greater_2_4_0_rc4(self):
        if self.spatial_version >= (2, 4, 1):
            return True
        else:
            # Spatialite 2.4.0-RC4 added AsGML and AsKML, however both
            # RC2 (shipped in popular Debian/Ubuntu packages) and RC4
            # report version as '2.4.0', so we fall back to feature detection
            try:
                self._get_spatialite_func("AsGML(GeomFromText('POINT(1 1)'))")
            except DatabaseError:
                return False
            return True

    @cached_property
    def gml(self):
        return 'AsGML' if self._version_greater_2_4_0_rc4 else None

    @cached_property
    def kml(self):
        return 'AsKML' if self._version_greater_2_4_0_rc4 else None

    @cached_property
    def geojson(self):
        return 'AsGeoJSON' if self.spatial_version >= (3, 0, 0) else None

    def check_aggregate_support(self, aggregate):
        """
        Checks if the given aggregate name is supported (that is, if it's
        in `self.valid_aggregates`).
        """
        super(SpatiaLiteOperations, self).check_aggregate_support(aggregate)
        agg_name = aggregate.__class__.__name__
        return agg_name in self.valid_aggregates

    def convert_extent(self, box):
        """
        Convert the polygon data received from Spatialite to min/max values.
        """
        shell = Geometry(box).shell
        xmin, ymin = shell[0][:2]
        xmax, ymax = shell[2][:2]
        return (xmin, ymin, xmax, ymax)

    def convert_geom(self, wkt, geo_field):
        """
        Converts geometry WKT returned from a SpatiaLite aggregate.
        """
        if wkt:
            return Geometry(wkt, geo_field.srid)
        else:
            return None

    def geo_db_type(self, f):
        """
        Returns None because geometry columnas are added via the
        `AddGeometryColumn` stored procedure on SpatiaLite.
        """
        return None

    def get_distance(self, f, value, lookup_type):
        """
        Returns the distance parameters for the given geometry field,
        lookup value, and lookup type.  SpatiaLite only supports regular
        cartesian-based queries (no spheroid/sphere calculations for point
        geometries like PostGIS).
        """
        if not value:
            return []
        value = value[0]
        if isinstance(value, Distance):
            if f.geodetic(self.connection):
                raise ValueError('SpatiaLite does not support distance queries on '
                                 'geometry fields with a geodetic coordinate system. '
                                 'Distance objects; use a numeric value of your '
                                 'distance in degrees instead.')
            else:
                dist_param = getattr(value, Distance.unit_attname(f.units_name(self.connection)))
        else:
            dist_param = value
        return [dist_param]

    def get_geom_placeholder(self, f, value):
        """
        Provides a proper substitution value for Geometries that are not in the
        SRID of the field.  Specifically, this routine will substitute in the
        Transform() and GeomFromText() function call(s).
        """
        def transform_value(value, srid):
            return not (value is None or value.srid == srid)
        if hasattr(value, 'expression'):
            if transform_value(value, f.srid):
                placeholder = '%s(%%s, %s)' % (self.transform, f.srid)
            else:
                placeholder = '%s'
            # No geometry value used for F expression, substitute in
            # the column name instead.
            return placeholder % self.get_expression_column(value)
        else:
            if transform_value(value, f.srid):
                # Adding Transform() to the SQL placeholder.
                return '%s(%s(%%s,%s), %s)' % (self.transform, self.from_text, value.srid, f.srid)
            else:
                return '%s(%%s,%s)' % (self.from_text, f.srid)

    def _get_spatialite_func(self, func):
        """
        Helper routine for calling SpatiaLite functions and returning
        their result.
        Any error occurring in this method should be handled by the caller.
        """
        cursor = self.connection._cursor()
        try:
            cursor.execute('SELECT %s' % func)
            row = cursor.fetchone()
        finally:
            cursor.close()
        return row[0]

    def geos_version(self):
        "Returns the version of GEOS used by SpatiaLite as a string."
        return self._get_spatialite_func('geos_version()')

    def proj4_version(self):
        "Returns the version of the PROJ.4 library used by SpatiaLite."
        return self._get_spatialite_func('proj4_version()')

    def spatialite_version(self):
        "Returns the SpatiaLite library version as a string."
        return self._get_spatialite_func('spatialite_version()')

    def spatialite_version_tuple(self):
        """
        Returns the SpatiaLite version as a tuple (version string, major,
        minor, subminor).
        """
        version = self.spatialite_version()

        m = self.version_regex.match(version)
        if m:
            major = int(m.group('major'))
            minor1 = int(m.group('minor1'))
            minor2 = int(m.group('minor2'))
        else:
            raise Exception('Could not parse SpatiaLite version string: %s' % version)

        return (version, major, minor1, minor2)

    def spatial_aggregate_sql(self, agg):
        """
        Returns the spatial aggregate SQL template and function for the
        given Aggregate instance.
        """
        agg_name = agg.__class__.__name__
        if not self.check_aggregate_support(agg):
            raise NotImplementedError('%s spatial aggregate is not implemented for this backend.' % agg_name)
        agg_name = agg_name.lower()
        if agg_name == 'union':
            agg_name += 'agg'
        sql_template = self.select % '%(function)s(%(field)s)'
        sql_function = getattr(self, agg_name)
        return sql_template, sql_function

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialiteGeometryColumns
        return SpatialiteGeometryColumns

    def spatial_ref_sys(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialiteSpatialRefSys
        return SpatialiteSpatialRefSys
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c) 2015 Cutting Edge QA

import os
import subprocess
import os.path
import urlparse
import urllib

from robot.api import logger

class ImageMagickKeywords(object):
    CONVERT_PATH = os.path.normpath(os.environ['MAGICK_HOME'] + "\\" + "convert.exe")
    COMPARE_PATH = os.path.normpath(os.environ['MAGICK_HOME'] + "\\" + "compare.exe")

    def image_self_check(self):
        try:
            os.environ['MAGICK_HOME']
        except:
            message = "Missing system variable 'MAGICK_HOME'"
            logger.error(message)
            raise AssertionError(message)

        if os.path.isfile(self.CONVERT_PATH):
            logger.info("Convert file exits")
        else:
            message = "Missing file convert.exe"
            raise AssertionError(message)

        if os.path.isfile(self.COMPARE_PATH):
            logger.info("Compare file exits")
        else:
            message = "Missing file compare.exe"
            raise AssertionError(message)

        argument_list = [self.CONVERT_PATH]
        try:
            procces = subprocess.Popen(argument_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # , shell=True
            procces.wait()
            logger.info("returnCode \t:" + str(procces.returncode))
            logger.info("stdout \t:" + str(procces.communicate()[0]))
            logger.info("stderr \t:" + str(procces.communicate()[1]))

        except OSError, e:
            logger.error(e)

    def _compare_image_files(self, file_1_path, file_2_path, gif_file_path=None, delta_file_path=None, metric="RMSE", embedded_gif=True, embedded_delta=False):
        file_1_path_normalized = os.path.normpath(file_1_path)
        file_2_path_normalized = os.path.normpath(file_2_path)
        if delta_file_path is not None:
            delta_file_path_normalized = os.path.normpath(delta_file_path)
        else:
            delta_file_path = os.path.dirname(file_1_path_normalized) + "\\" + os.path.splitext(os.path.basename(file_1_path_normalized))[0] + "_" + \
                              os.path.splitext(os.path.basename(file_2_path_normalized))[0] + "_delta.png"
            delta_file_path_normalized = os.path.normpath(delta_file_path)

        if gif_file_path is not None:
            gif_file_path_normalized = os.path.normpath(gif_file_path)
        else:
            gif_file_path = os.path.dirname(file_1_path_normalized) + "\\" + os.path.splitext(os.path.basename(file_1_path_normalized))[0] + "_" + \
                            os.path.splitext(os.path.basename(file_2_path_normalized))[0] + ".gif"
            gif_file_path_normalized = os.path.normpath(gif_file_path)
        if os.path.isfile(file_1_path_normalized) and os.path.isfile(file_2_path_normalized):
            argument_list = [self.COMPARE_PATH, "-metric", metric, file_1_path_normalized, file_2_path_normalized, delta_file_path_normalized]
            process = subprocess.Popen(argument_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            process.wait()
            delta_percent = process.communicate()[1].split("(")[1].replace(")", "")

            if gif_file_path is not None:
                self.create_gif_from_three_files(gif_file_path_normalized, file_1_path_normalized, file_2_path_normalized, delta_file_path_normalized,
                                                 embedded=embedded_gif)
            if delta_file_path is None:
                os.remove(delta_file_path_normalized)
            else:
                if embedded_delta:
                    self._embed_screenshot(delta_file_path_normalized)

            return float(delta_percent) * float(100), delta_file_path_normalized, gif_file_path_normalized

    def compare_image_files(self, file_1_path, file_2_path, gif_file_path=None, delta_file_path=None, metric="RMSE", embedded_gif=True, embedded_delta=False):
        return self._compare_image_files(file_1_path, file_2_path, gif_file_path, delta_file_path, metric, embedded_gif, embedded_delta)[0]

    def image_should_be_difference_less_then(self, file_1_path, file_2_path, difference_percent=1, gif_file_path=None, delta_file_path=None, embedded_gif=True,
                                             embedded_delta=False):
        """difference_percent test to 0 mean both images are identical """
        results = self._compare_image_files(file_1_path, file_2_path, gif_file_path, delta_file_path, embedded_gif=embedded_gif, embedded_delta=embedded_delta)
        if float(results[0]) > float(difference_percent):
            message = "Difference between files is greater then expected actual %.2f > %.2f expected percent" % (float(results[0]), float(difference_percent))
            raise AssertionError(message)
        else:
            logger.info("Image check successful ")
        return results[0]

    def create_gif_from_three_files(self, gif_file_path, file_1_path, file_2_path, file_3_path, delay_ms=100, loop=0, embedded=True):
        files_list = [file_1_path, file_2_path, file_3_path]
        self.create_gif_from_list_of_files(gif_file_path, files_list, delay_ms, loop, embedded)

    def create_gif_from_list_of_files(self, gif_file_path, files_list_path, delay_ms=100, loop=0, embedded=True):
        files_list_path_normalized = []
        for singleFile in files_list_path:
            files_list_path_normalized.append(os.path.normpath(singleFile))
        gif_file_path_normalized = os.path.normpath(gif_file_path)
        if all(os.path.isfile(singleFile) == True for singleFile in files_list_path_normalized):
            argument_list = [self.CONVERT_PATH, "-delay", str(delay_ms), "-loop", str(loop)]
            argument_list.extend(files_list_path_normalized)
            argument_list.append(gif_file_path_normalized)
            process = subprocess.Popen(argument_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            process.wait()
            if embedded:
                self._embed_screenshot(gif_file_path_normalized)

        else:
            message = "Gif Creation failed."
            for singleFile in files_list_path_normalized:
                if os.path.isfile(singleFile):
                    message += "File missing %s." % singleFile
            raise AssertionError(message)

    @staticmethod
    def _embed_screenshot(path, level="INFO", width="800px"):
        link = urlparse.urljoin('file:', urllib.pathname2url(os.path.normpath(path)))
        logger.write('' % (link, link, width), level, html=True)
<commit_msg>minor bug fix, add more error handling<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c) 2015 Cutting Edge QA

import os
import subprocess
import os.path
import urlparse
import urllib

from robot.api import logger

class ImageMagickKeywords(object):
    CONVERT_PATH = os.path.normpath(os.environ['MAGICK_HOME'] + "\\" + "convert.exe")
    COMPARE_PATH = os.path.normpath(os.environ['MAGICK_HOME'] + "\\" + "compare.exe")

    def image_self_check(self):
        try:
            os.environ['MAGICK_HOME']
        except:
            message = "Missing system variable 'MAGICK_HOME'"
            logger.error(message)
            raise AssertionError(message)

        if os.path.isfile(self.CONVERT_PATH):
            logger.info("Convert file exits")
        else:
            message = "Missing file convert.exe"
            raise AssertionError(message)

        if os.path.isfile(self.COMPARE_PATH):
            logger.info("Compare file exits")
        else:
            message = "Missing file compare.exe"
            raise AssertionError(message)

        argument_list = [self.CONVERT_PATH, "--version"]
        try:
            procces = subprocess.Popen(argument_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # , shell=True
            procces.wait()
            output = procces.communicate()
            logger.info("returnCode \t:" + str(procces.returncode))
            if procces.returncode != 0:
                message = "processing failed msg= %s" % str(output)
                raise AssertionError(message)
            logger.info("stdout \t:" + str(output[0]))
            logger.info("stderr \t:" + str(output[1]))

        except OSError, e:
            logger.error(e)

    def _compare_image_files(self, file_1_path, file_2_path, gif_file_path=None, delta_file_path=None, metric="RMSE", embedded_gif=True, embedded_delta=False):
        file_1_path_normalized = os.path.normpath(file_1_path)
        file_2_path_normalized = os.path.normpath(file_2_path)
        if delta_file_path is not None:
            delta_file_path_normalized = os.path.normpath(delta_file_path)
        else:
            delta_file_path = os.path.dirname(file_1_path_normalized) + "\\" + os.path.splitext(os.path.basename(file_1_path_normalized))[0] + "_" + \
                              os.path.splitext(os.path.basename(file_2_path_normalized))[0] + "_delta.png"
            delta_file_path_normalized = os.path.normpath(delta_file_path)

        if gif_file_path is not None:
            gif_file_path_normalized = os.path.normpath(gif_file_path)
        else:
            gif_file_path = os.path.dirname(file_1_path_normalized) + "\\" + os.path.splitext(os.path.basename(file_1_path_normalized))[0] + "_" + \
                            os.path.splitext(os.path.basename(file_2_path_normalized))[0] + ".gif"
            gif_file_path_normalized = os.path.normpath(gif_file_path)
        if os.path.isfile(file_1_path_normalized) and os.path.isfile(file_2_path_normalized):
            argument_list = [self.COMPARE_PATH, "-metric", metric, file_1_path_normalized, file_2_path_normalized, delta_file_path_normalized]
            process = subprocess.Popen(argument_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            process.wait()
            output = process.communicate()
            if "error" in str(output):
                message = "processing failed msg= %s" % str(output)
                raise AssertionError(message)

            delta_percent = output[1].split("(")[1].replace(")", "")

            if gif_file_path is not None:
                self.create_gif_from_three_files(gif_file_path_normalized, file_1_path_normalized, file_2_path_normalized, delta_file_path_normalized,
                                                 embedded=embedded_gif)
            if delta_file_path is None:
                TODO = True
                # TODO maybe remove files
            else:
                if embedded_delta:
                    self._embed_screenshot(delta_file_path_normalized)

            return float(delta_percent) * float(100), delta_file_path_normalized, gif_file_path_normalized

    def compare_image_files(self, file_1_path, file_2_path, gif_file_path=None, delta_file_path=None, metric="RMSE", embedded_gif=True, embedded_delta=False):
        results = self._compare_image_files(file_1_path, file_2_path, gif_file_path, delta_file_path, metric, embedded_gif, embedded_delta)

        return results[0]

    def image_should_be_difference_less_then(self, file_1_path, file_2_path, difference_percent=1, gif_file_path=None, delta_file_path=None, embedded_gif=True,
                                             embedded_delta=False):
        """difference_percent test to 0 mean both images are identical """
        results = self._compare_image_files(file_1_path, file_2_path, gif_file_path, delta_file_path, embedded_gif=embedded_gif, embedded_delta=embedded_delta)
        print str(results)
        if float(results[0]) > float(difference_percent):
            message = "Difference between files is greater then expected actual %.2f > %.2f expected percent" % (float(results[0]), float(difference_percent))
            raise AssertionError(message)
        else:
            logger.info("Image check successful ")
        return results[0]

    def create_gif_from_three_files(self, gif_file_path, file_1_path, file_2_path, file_3_path, delay_ms=100, loop=0, embedded=True):
        files_list = [file_1_path, file_2_path, file_3_path]
        self.create_gif_from_list_of_files(gif_file_path, files_list, delay_ms, loop, embedded)

    def create_gif_from_list_of_files(self, gif_file_path, files_list_path, delay_ms=100, loop=0, embedded=True):
        files_list_path_normalized = []
        for singleFile in files_list_path:
            files_list_path_normalized.append(os.path.normpath(singleFile))
        gif_file_path_normalized = os.path.normpath(gif_file_path)
        if all(os.path.isfile(singleFile) == True for singleFile in files_list_path_normalized):
            argument_list = [self.CONVERT_PATH, "-delay", str(delay_ms), "-loop", str(loop)]
            argument_list.extend(files_list_path_normalized)
            argument_list.append(gif_file_path_normalized)
            process = subprocess.Popen(argument_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            process.wait()
            if embedded:
                self._embed_screenshot(gif_file_path_normalized)

        else:
            message = "Gif Creation failed."
            for singleFile in files_list_path_normalized:
                if os.path.isfile(singleFile):
                    message += "File missing %s." % singleFile
            raise AssertionError(message)

    @staticmethod
    def _embed_screenshot(path, level="INFO", width="800px"):
        link = urlparse.urljoin('file:', urllib.pathname2url(os.path.normpath(path)))
        logger.write('' % (link, link, width), level, html=True)
<|endoftext|>