<commit_before># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------
import sys
import os
import tempfile
import shutil
import zipfile
import traceback
import hashlib
from subprocess import check_output, STDOUT, CalledProcessError
from six.moves.urllib.parse import urlparse  # pylint: disable=import-error
from collections import OrderedDict

import requests
from wheel.install import WHEEL_INFO_RE
from pkg_resources import parse_version

from knack.log import get_logger

from azure.cli.core.util import CLIError
from azure.cli.core.extension import (extension_exists, get_extension_path, get_extensions,
                                      get_extension, ext_compat_with_cli, EXT_METADATA_ISPREVIEW,
                                      WheelExtension, ExtensionNotInstalledException)
from azure.cli.core.telemetry import set_extension_management_detail

from ._homebrew_patch import HomebrewPipPatch
from ._index import get_index_extensions
from ._resolve import resolve_from_index, NoExtensionCandidatesError

logger = get_logger(__name__)

OUT_KEY_NAME = 'name'
OUT_KEY_VERSION = 'version'
OUT_KEY_TYPE = 'extensionType'
OUT_KEY_METADATA = 'metadata'

IS_WINDOWS = sys.platform.lower() in ['windows', 'win32']
LIST_FILE_PATH = os.path.join(os.sep, 'etc', 'apt', 'sources.list.d', 'azure-cli.list')
LSB_RELEASE_FILE = os.path.join(os.sep, 'etc', 'lsb-release')

def _run_pip(pip_exec_args):
    cmd = [sys.executable, '-m', 'pip'] + pip_exec_args + ['-vv', '--disable-pip-version-check', '--no-cache-dir']
    logger.debug('Running: %s', cmd)
    try:
        log_output = check_output(cmd, stderr=STDOUT, universal_newlines=True)
        logger.debug(log_output)
        returncode = 0
    except CalledProcessError as e:
        logger.debug(e.output)
        logger.debug(e)
        returncode = e.returncode
    return returncode

def _whl_download_from_url(url_parse_result, ext_file):
    from azure.cli.core.util import should_disable_connection_verify
    url = url_parse_result.geturl()
    r = requests.get(url, stream=True, verify=(not should_disable_connection_verify()))
    if r.status_code != 200:
        raise CLIError("Request to {} failed with {}".format(url, r.status_code))
    with open(ext_file, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # ignore keep-alive new chunks
                f.write(chunk)

def _validate_whl_cli_compat(azext_metadata):
    is_compatible, cli_core_version, min_required, max_required = ext_compat_with_cli(azext_metadata)
    logger.debug("Extension compatibility result: is_compatible=%s cli_core_version=%s min_required=%s "
                 "max_required=%s", is_compatible, cli_core_version, min_required, max_required)
    if not is_compatible:
        min_max_msg_fmt = "The extension is not compatible with this version of the CLI.\n" \
                          "You have CLI core version {} and this extension " \
                          "requires ".format(cli_core_version)
        if min_required and max_required:
            min_max_msg_fmt += 'a min of {} and max of {}.'.format(min_required, max_required)
        elif min_required:
            min_max_msg_fmt += 'a min of {}.'.format(min_required)
        elif max_required:
            min_max_msg_fmt += 'a max of {}.'.format(max_required)
        raise CLIError(min_max_msg_fmt)

def _validate_whl_extension(ext_file):
    tmp_dir = tempfile.mkdtemp()
    zip_ref = zipfile.ZipFile(ext_file, 'r')
    zip_ref.extractall(tmp_dir)
    zip_ref.close()
    azext_metadata = WheelExtension.get_azext_metadata(tmp_dir)
    shutil.rmtree(tmp_dir)
    _validate_whl_cli_compat(azext_metadata)

def _add_whl_ext(source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None):  # pylint: disable=too-many-statements
    if not source.endswith('.whl'):
        raise ValueError('Unknown extension type. Only Python wheels are supported.')
    url_parse_result = urlparse(source)
    is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
    logger.debug('Extension source is url? %s', is_url)
    whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
    parsed_filename = WHEEL_INFO_RE(whl_filename)
    # Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this.
    extension_name = parsed_filename.groupdict().get('name').replace('_', '-') if parsed_filename else None
    if not extension_name:
        raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
    if extension_exists(extension_name):
        raise CLIError('The extension {} already exists.'.format(extension_name))
    ext_file = None
    if is_url:
        # Download from URL
        tmp_dir = tempfile.mkdtemp()
        ext_file = os.path.join(tmp_dir, whl_filename)
        logger.debug('Downloading %s to %s', source, ext_file)
        try:
            _whl_download_from_url(url_parse_result, ext_file)
        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
            raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
        logger.debug('Downloaded to %s', ext_file)
    else:
        # Get file path
        ext_file = os.path.realpath(os.path.expanduser(source))
        if not os.path.isfile(ext_file):
            raise CLIError("File {} not found.".format(source))
    # Validate the extension
    logger.debug('Validating the extension %s', ext_file)
    if ext_sha256:
        valid_checksum, computed_checksum = is_valid_sha256sum(ext_file, ext_sha256)
        if valid_checksum:
            logger.debug("Checksum of %s is OK", ext_file)
        else:
            logger.debug("Invalid checksum for %s. Expected '%s', computed '%s'.",
                         ext_file, ext_sha256, computed_checksum)
            raise CLIError("The checksum of the extension does not match the expected value. "
                           "Use --debug for more information.")
    try:
        _validate_whl_extension(ext_file)
    except AssertionError:
        logger.debug(traceback.format_exc())
        raise CLIError('The extension is invalid. Use --debug for more information.')
    except CLIError as e:
        raise e
    logger.debug('Validation successful on %s', ext_file)
    # Check for distro consistency
    check_distro_consistency()
    # Install with pip
    extension_path = get_extension_path(extension_name)
    pip_args = ['install', '--target', extension_path, ext_file]

    if pip_proxy:
        pip_args = pip_args + ['--proxy', pip_proxy]
    if pip_extra_index_urls:
        for extra_index_url in pip_extra_index_urls:
            pip_args = pip_args + ['--extra-index-url', extra_index_url]

    logger.debug('Executing pip with args: %s', pip_args)
    with HomebrewPipPatch():
        pip_status_code = _run_pip(pip_args)
    if pip_status_code > 0:
        logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
        shutil.rmtree(extension_path, ignore_errors=True)
        raise CLIError('An error occurred. Pip failed with status code {}. '
                       'Use --debug for more information.'.format(pip_status_code))
    # Save the whl we used to install the extension in the extension dir.
    dst = os.path.join(extension_path, whl_filename)
    shutil.copyfile(ext_file, dst)
    logger.debug('Saved the whl to %s', dst)

def is_valid_sha256sum(a_file, expected_sum):
    sha256 = hashlib.sha256()
    with open(a_file, 'rb') as f:
        sha256.update(f.read())
    computed_hash = sha256.hexdigest()
    return expected_sum == computed_hash, computed_hash

def _augment_telemetry_with_ext_info(extension_name):
    # The extension must be available before calling this otherwise we can't get the version from metadata
    if not extension_name:
        return
    try:
        ext = get_extension(extension_name)
        ext_version = ext.version
        set_extension_management_detail(extension_name, ext_version)
    except Exception:  # nopa pylint: disable=broad-except
        # Don't error on telemetry
        pass

def add_extension(source=None, extension_name=None, index_url=None, yes=None,  # pylint: disable=unused-argument
                  pip_extra_index_urls=None, pip_proxy=None):
    ext_sha256 = None
    if extension_name:
        if extension_exists(extension_name):
            raise CLIError('The extension {} already exists.'.format(extension_name))
        try:
            source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url)
        except NoExtensionCandidatesError as err:
            logger.debug(err)
            raise CLIError("No matching extensions for '{}'. Use --debug for more information.".format(extension_name))
    _add_whl_ext(source, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
    _augment_telemetry_with_ext_info(extension_name)
    try:
        if extension_name and get_extension(extension_name).preview:
            logger.warning("The installed extension '%s' is in preview.", extension_name)
    except ExtensionNotInstalledException:
        pass

def remove_extension(extension_name):
    try:
        # Get the extension and it will raise an error if it doesn't exist
        get_extension(extension_name)
        # We call this just before we remove the extension so we can get the metadata before it is gone
        _augment_telemetry_with_ext_info(extension_name)
        shutil.rmtree(get_extension_path(extension_name))
    except ExtensionNotInstalledException as e:
        raise CLIError(e)

def list_extensions():
    return [{OUT_KEY_NAME: ext.name, OUT_KEY_VERSION: ext.version, OUT_KEY_TYPE: ext.ext_type}
            for ext in get_extensions()]

def show_extension(extension_name):
    try:
        extension = get_extension(extension_name)
        return {OUT_KEY_NAME: extension.name,
                OUT_KEY_VERSION: extension.version,
                OUT_KEY_TYPE: extension.ext_type,
                OUT_KEY_METADATA: extension.metadata}
    except ExtensionNotInstalledException as e:
        raise CLIError(e)

def update_extension(extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None):
    try:
        ext = get_extension(extension_name)
        cur_version = ext.get_version()
        try:
            download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url)
        except NoExtensionCandidatesError as err:
            logger.debug(err)
            raise CLIError("No updates available for '{}'. Use --debug for more information.".format(extension_name))
        # Copy current version of extension to tmp directory in case we need to restore it after a failed install.
        backup_dir = os.path.join(tempfile.mkdtemp(), extension_name)
        extension_path = get_extension_path(extension_name)
        logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir)
        shutil.copytree(extension_path, backup_dir)
        # Remove current version of the extension
        shutil.rmtree(extension_path)
        # Install newer version
        try:
            _add_whl_ext(download_url, ext_sha256=ext_sha256,
                         pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
            logger.debug('Deleting backup of old extension at %s', backup_dir)
            shutil.rmtree(backup_dir)
            # This gets the metadata for the extension *after* the update
            _augment_telemetry_with_ext_info(extension_name)
        except Exception as err:
            logger.error('An error occurred whilst updating.')
            logger.error(err)
            logger.debug('Copying %s to %s', backup_dir, extension_path)
            shutil.copytree(backup_dir, extension_path)
            raise CLIError('Failed to update. Rolled {} back to {}.'.format(extension_name, cur_version))
    except ExtensionNotInstalledException as e:
        raise CLIError(e)

def list_available_extensions(index_url=None, show_details=False):
    index_data = get_index_extensions(index_url=index_url)
    if show_details:
        return index_data
    installed_extensions = get_extensions()
    installed_extension_names = [e.name for e in installed_extensions]
    results = []
    for name, items in OrderedDict(sorted(index_data.items())).items():
        latest = sorted(items, key=lambda c: parse_version(c['metadata']['version']), reverse=True)[0]
        installed = False
        if name in installed_extension_names:
            installed = True
            if parse_version(latest['metadata']['version']) > parse_version(get_extension(name).version):
                installed = str(True) + ' (upgrade available)'
        results.append({
            'name': name,
            'version': latest['metadata']['version'],
            'summary': latest['metadata']['summary'],
            'preview': latest['metadata'].get(EXT_METADATA_ISPREVIEW, False),
            'installed': installed
        })
    return results

def get_lsb_release():
    try:
        with open(LSB_RELEASE_FILE, 'r') as lr:
            lsb = lr.readlines()
            desc = lsb[2]
            desc_split = desc.split('=')
            rel = desc_split[1]
            return rel.strip()
    except Exception:  # pylint: disable=broad-except
        return None

def check_distro_consistency():
    if IS_WINDOWS:
        return

    try:
        logger.debug('Linux distro check: Reading from: %s', LIST_FILE_PATH)

        with open(LIST_FILE_PATH, 'r') as list_file:
            package_source = list_file.read()
            stored_linux_dist_name = package_source.split(" ")[3]
            logger.debug('Linux distro check: Found in list file: %s', stored_linux_dist_name)
            current_linux_dist_name = get_lsb_release()
            logger.debug('Linux distro check: Reported by API: %s', current_linux_dist_name)

    except Exception as err:  # pylint: disable=broad-except
        current_linux_dist_name = None
        stored_linux_dist_name = None
        logger.debug('Linux distro check: An error occurred while checking '
                     'linux distribution version source list consistency.')
        logger.debug(err)

    if current_linux_dist_name != stored_linux_dist_name:
        logger.debug("Linux distro check: Mismatch distribution "
                     "name in %s file", LIST_FILE_PATH)
        logger.debug("Linux distro check: If command fails, install the appropriate package "
                     "for your distribution or change the above file accordingly.")
        logger.debug("Linux distro check: %s has '%s', current distro is '%s'",
                     LIST_FILE_PATH, stored_linux_dist_name, current_linux_dist_name)
<commit_msg>Bug fix: If extension doesn't have a version, don't parse version (#5989)<commit_after># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------
import sys
import os
import tempfile
import shutil
import zipfile
import traceback
import hashlib
from subprocess import check_output, STDOUT, CalledProcessError
from six.moves.urllib.parse import urlparse  # pylint: disable=import-error
from collections import OrderedDict

import requests
from wheel.install import WHEEL_INFO_RE
from pkg_resources import parse_version

from knack.log import get_logger

from azure.cli.core.util import CLIError
from azure.cli.core.extension import (extension_exists, get_extension_path, get_extensions,
                                      get_extension, ext_compat_with_cli, EXT_METADATA_ISPREVIEW,
                                      WheelExtension, ExtensionNotInstalledException)
from azure.cli.core.telemetry import set_extension_management_detail

from ._homebrew_patch import HomebrewPipPatch
from ._index import get_index_extensions
from ._resolve import resolve_from_index, NoExtensionCandidatesError

logger = get_logger(__name__)

OUT_KEY_NAME = 'name'
OUT_KEY_VERSION = 'version'
OUT_KEY_TYPE = 'extensionType'
OUT_KEY_METADATA = 'metadata'

IS_WINDOWS = sys.platform.lower() in ['windows', 'win32']
LIST_FILE_PATH = os.path.join(os.sep, 'etc', 'apt', 'sources.list.d', 'azure-cli.list')
LSB_RELEASE_FILE = os.path.join(os.sep, 'etc', 'lsb-release')

def _run_pip(pip_exec_args):
    cmd = [sys.executable, '-m', 'pip'] + pip_exec_args + ['-vv', '--disable-pip-version-check', '--no-cache-dir']
    logger.debug('Running: %s', cmd)
    try:
        log_output = check_output(cmd, stderr=STDOUT, universal_newlines=True)
        logger.debug(log_output)
        returncode = 0
    except CalledProcessError as e:
        logger.debug(e.output)
        logger.debug(e)
        returncode = e.returncode
    return returncode

def _whl_download_from_url(url_parse_result, ext_file):
    from azure.cli.core.util import should_disable_connection_verify
    url = url_parse_result.geturl()
    r = requests.get(url, stream=True, verify=(not should_disable_connection_verify()))
    if r.status_code != 200:
        raise CLIError("Request to {} failed with {}".format(url, r.status_code))
    with open(ext_file, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # ignore keep-alive new chunks
                f.write(chunk)

def _validate_whl_cli_compat(azext_metadata):
    is_compatible, cli_core_version, min_required, max_required = ext_compat_with_cli(azext_metadata)
    logger.debug("Extension compatibility result: is_compatible=%s cli_core_version=%s min_required=%s "
                 "max_required=%s", is_compatible, cli_core_version, min_required, max_required)
    if not is_compatible:
        min_max_msg_fmt = "The extension is not compatible with this version of the CLI.\n" \
                          "You have CLI core version {} and this extension " \
                          "requires ".format(cli_core_version)
        if min_required and max_required:
            min_max_msg_fmt += 'a min of {} and max of {}.'.format(min_required, max_required)
        elif min_required:
            min_max_msg_fmt += 'a min of {}.'.format(min_required)
        elif max_required:
            min_max_msg_fmt += 'a max of {}.'.format(max_required)
        raise CLIError(min_max_msg_fmt)

def _validate_whl_extension(ext_file):
    tmp_dir = tempfile.mkdtemp()
    zip_ref = zipfile.ZipFile(ext_file, 'r')
    zip_ref.extractall(tmp_dir)
    zip_ref.close()
    azext_metadata = WheelExtension.get_azext_metadata(tmp_dir)
    shutil.rmtree(tmp_dir)
    _validate_whl_cli_compat(azext_metadata)

def _add_whl_ext(source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None):  # pylint: disable=too-many-statements
    if not source.endswith('.whl'):
        raise ValueError('Unknown extension type. Only Python wheels are supported.')
    url_parse_result = urlparse(source)
    is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
    logger.debug('Extension source is url? %s', is_url)
    whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
    parsed_filename = WHEEL_INFO_RE(whl_filename)
    # Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this.
    extension_name = parsed_filename.groupdict().get('name').replace('_', '-') if parsed_filename else None
    if not extension_name:
        raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
    if extension_exists(extension_name):
        raise CLIError('The extension {} already exists.'.format(extension_name))
    ext_file = None
    if is_url:
        # Download from URL
        tmp_dir = tempfile.mkdtemp()
        ext_file = os.path.join(tmp_dir, whl_filename)
        logger.debug('Downloading %s to %s', source, ext_file)
        try:
            _whl_download_from_url(url_parse_result, ext_file)
        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
            raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
        logger.debug('Downloaded to %s', ext_file)
    else:
        # Get file path
        ext_file = os.path.realpath(os.path.expanduser(source))
        if not os.path.isfile(ext_file):
            raise CLIError("File {} not found.".format(source))
    # Validate the extension
    logger.debug('Validating the extension %s', ext_file)
    if ext_sha256:
        valid_checksum, computed_checksum = is_valid_sha256sum(ext_file, ext_sha256)
        if valid_checksum:
            logger.debug("Checksum of %s is OK", ext_file)
        else:
            logger.debug("Invalid checksum for %s. Expected '%s', computed '%s'.",
                         ext_file, ext_sha256, computed_checksum)
            raise CLIError("The checksum of the extension does not match the expected value. "
                           "Use --debug for more information.")
    try:
        _validate_whl_extension(ext_file)
    except AssertionError:
        logger.debug(traceback.format_exc())
        raise CLIError('The extension is invalid. Use --debug for more information.')
    except CLIError as e:
        raise e
    logger.debug('Validation successful on %s', ext_file)
    # Check for distro consistency
    check_distro_consistency()
    # Install with pip
    extension_path = get_extension_path(extension_name)
    pip_args = ['install', '--target', extension_path, ext_file]

    if pip_proxy:
        pip_args = pip_args + ['--proxy', pip_proxy]
    if pip_extra_index_urls:
        for extra_index_url in pip_extra_index_urls:
            pip_args = pip_args + ['--extra-index-url', extra_index_url]

    logger.debug('Executing pip with args: %s', pip_args)
    with HomebrewPipPatch():
        pip_status_code = _run_pip(pip_args)
    if pip_status_code > 0:
        logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
        shutil.rmtree(extension_path, ignore_errors=True)
        raise CLIError('An error occurred. Pip failed with status code {}. '
                       'Use --debug for more information.'.format(pip_status_code))
    # Save the whl we used to install the extension in the extension dir.
    dst = os.path.join(extension_path, whl_filename)
    shutil.copyfile(ext_file, dst)
    logger.debug('Saved the whl to %s', dst)

def is_valid_sha256sum(a_file, expected_sum):
    sha256 = hashlib.sha256()
    with open(a_file, 'rb') as f:
        sha256.update(f.read())
    computed_hash = sha256.hexdigest()
    return expected_sum == computed_hash, computed_hash

def _augment_telemetry_with_ext_info(extension_name):
    # The extension must be available before calling this otherwise we can't get the version from metadata
    if not extension_name:
        return
    try:
        ext = get_extension(extension_name)
        ext_version = ext.version
        set_extension_management_detail(extension_name, ext_version)
    except Exception:  # nopa pylint: disable=broad-except
        # Don't error on telemetry
        pass

def add_extension(source=None, extension_name=None, index_url=None, yes=None,  # pylint: disable=unused-argument
                  pip_extra_index_urls=None, pip_proxy=None):
    ext_sha256 = None
    if extension_name:
        if extension_exists(extension_name):
            raise CLIError('The extension {} already exists.'.format(extension_name))
        try:
            source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url)
        except NoExtensionCandidatesError as err:
            logger.debug(err)
            raise CLIError("No matching extensions for '{}'. Use --debug for more information.".format(extension_name))
    _add_whl_ext(source, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
    _augment_telemetry_with_ext_info(extension_name)
    try:
        if extension_name and get_extension(extension_name).preview:
            logger.warning("The installed extension '%s' is in preview.", extension_name)
    except ExtensionNotInstalledException:
        pass

def remove_extension(extension_name):
    try:
        # Get the extension and it will raise an error if it doesn't exist
        get_extension(extension_name)
        # We call this just before we remove the extension so we can get the metadata before it is gone
        _augment_telemetry_with_ext_info(extension_name)
        shutil.rmtree(get_extension_path(extension_name))
    except ExtensionNotInstalledException as e:
        raise CLIError(e)

def list_extensions():
    return [{OUT_KEY_NAME: ext.name, OUT_KEY_VERSION: ext.version, OUT_KEY_TYPE: ext.ext_type}
            for ext in get_extensions()]

def show_extension(extension_name):
    try:
        extension = get_extension(extension_name)
        return {OUT_KEY_NAME: extension.name,
                OUT_KEY_VERSION: extension.version,
                OUT_KEY_TYPE: extension.ext_type,
                OUT_KEY_METADATA: extension.metadata}
    except ExtensionNotInstalledException as e:
        raise CLIError(e)

def update_extension(extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None):
    try:
        ext = get_extension(extension_name)
        cur_version = ext.get_version()
        try:
            download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url)
        except NoExtensionCandidatesError as err:
            logger.debug(err)
            raise CLIError("No updates available for '{}'. Use --debug for more information.".format(extension_name))
        # Copy current version of extension to tmp directory in case we need to restore it after a failed install.
        backup_dir = os.path.join(tempfile.mkdtemp(), extension_name)
        extension_path = get_extension_path(extension_name)
        logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir)
        shutil.copytree(extension_path, backup_dir)
        # Remove current version of the extension
        shutil.rmtree(extension_path)
        # Install newer version
        try:
            _add_whl_ext(download_url, ext_sha256=ext_sha256,
                         pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
            logger.debug('Deleting backup of old extension at %s', backup_dir)
            shutil.rmtree(backup_dir)
            # This gets the metadata for the extension *after* the update
            _augment_telemetry_with_ext_info(extension_name)
        except Exception as err:
            logger.error('An error occurred whilst updating.')
            logger.error(err)
            logger.debug('Copying %s to %s', backup_dir, extension_path)
            shutil.copytree(backup_dir, extension_path)
            raise CLIError('Failed to update. Rolled {} back to {}.'.format(extension_name, cur_version))
    except ExtensionNotInstalledException as e:
        raise CLIError(e)

def list_available_extensions(index_url=None, show_details=False):
    index_data = get_index_extensions(index_url=index_url)
    if show_details:
        return index_data
    installed_extensions = get_extensions()
    installed_extension_names = [e.name for e in installed_extensions]
    results = []
    for name, items in OrderedDict(sorted(index_data.items())).items():
        latest = sorted(items, key=lambda c: parse_version(c['metadata']['version']), reverse=True)[0]
        installed = False
        if name in installed_extension_names:
            installed = True
            ext_version = get_extension(name).version
            if ext_version and parse_version(latest['metadata']['version']) > parse_version(ext_version):
                installed = str(True) + ' (upgrade available)'
        results.append({
            'name': name,
            'version': latest['metadata']['version'],
            'summary': latest['metadata']['summary'],
            'preview': latest['metadata'].get(EXT_METADATA_ISPREVIEW, False),
            'installed': installed
        })
    return results

def get_lsb_release():
    try:
        with open(LSB_RELEASE_FILE, 'r') as lr:
            lsb = lr.readlines()
            desc = lsb[2]
            desc_split = desc.split('=')
            rel = desc_split[1]
            return rel.strip()
    except Exception:  # pylint: disable=broad-except
        return None

def check_distro_consistency():
    if IS_WINDOWS:
        return

    try:
        logger.debug('Linux distro check: Reading from: %s', LIST_FILE_PATH)

        with open(LIST_FILE_PATH, 'r') as list_file:
            package_source = list_file.read()
            stored_linux_dist_name = package_source.split(" ")[3]
            logger.debug('Linux distro check: Found in list file: %s', stored_linux_dist_name)
            current_linux_dist_name = get_lsb_release()
            logger.debug('Linux distro check: Reported by API: %s', current_linux_dist_name)

    except Exception as err:  # pylint: disable=broad-except
        current_linux_dist_name = None
        stored_linux_dist_name = None
        logger.debug('Linux distro check: An error occurred while checking '
                     'linux distribution version source list consistency.')
        logger.debug(err)

    if current_linux_dist_name != stored_linux_dist_name:
        logger.debug("Linux distro check: Mismatch distribution "
                     "name in %s file", LIST_FILE_PATH)
        logger.debug("Linux distro check: If command fails, install the appropriate package "
                     "for your distribution or change the above file accordingly.")
        logger.debug("Linux distro check: %s has '%s', current distro is '%s'",
                     LIST_FILE_PATH, stored_linux_dist_name, current_linux_dist_name)
<|endoftext|>


<commit_before>import numpy as np

# the type of float to use throughout the session.
_FLOATX = 'float32'
_EPSILON = 10e-8
_UID_PREFIXES = {}
_IMAGE_DIM_ORDERING = 'th'

def epsilon():
    '''Returns the value of the fuzz
    factor used in numeric expressions.
    '''
    return _EPSILON

def set_epsilon(e):
    '''Sets the value of the fuzz
    factor used in numeric expressions.
    '''
    global _EPSILON
    _EPSILON = e

def floatx():
    '''Returns the default float type, as a string
    (e.g. 'float16', 'float32', 'float64').
    '''
    return _FLOATX

def set_floatx(floatx):
    global _FLOATX
    if floatx not in {'float16', 'float32', 'float64'}:
        raise Exception('Unknown floatx type: ' + str(floatx))
    _FLOATX = str(floatx)

def cast_to_floatx(x):
    '''Cast a Numpy array to floatx.
    '''
    return np.asarray(x, dtype=_FLOATX)

def image_dim_ordering():
    '''Returns the image dimension ordering
    convention ('th' or 'tf').
    '''
    return _IMAGE_DIM_ORDERING

def set_image_dim_ordering(dim_ordering):
    '''Sets the value of the image dimension
    ordering convention ('th' or 'tf').
    '''
    global _IMAGE_DIM_ORDERING
    if dim_ordering not in {'tf', 'th'}:
        raise Exception('Unknown dim_ordering:', dim_ordering)
    _IMAGE_DIM_ORDERING = str(dim_ordering)

def get_uid(prefix=''):
    if prefix not in _UID_PREFIXES:
        _UID_PREFIXES[prefix] = 1
        return 1
    else:
        _UID_PREFIXES[prefix] += 1
        return _UID_PREFIXES[prefix]
<commit_msg>Use defaultdict for _UID_PREFIXES (#3087)<commit_after>import numpy as np

from collections import defaultdict

# the type of float to use throughout the session.
_FLOATX = 'float32'
_EPSILON = 10e-8
_UID_PREFIXES = defaultdict(int)
_IMAGE_DIM_ORDERING = 'th'

def epsilon():
    '''Returns the value of the fuzz
    factor used in numeric expressions.
    '''
    return _EPSILON

def set_epsilon(e):
    '''Sets the value of the fuzz
    factor used in numeric expressions.
    '''
    global _EPSILON
    _EPSILON = e

def floatx():
    '''Returns the default float type, as a string
    (e.g. 'float16', 'float32', 'float64').
    '''
    return _FLOATX

def set_floatx(floatx):
    global _FLOATX
    if floatx not in {'float16', 'float32', 'float64'}:
        raise Exception('Unknown floatx type: ' + str(floatx))
    _FLOATX = str(floatx)

def cast_to_floatx(x):
    '''Cast a Numpy array to floatx.
    '''
    return np.asarray(x, dtype=_FLOATX)

def image_dim_ordering():
    '''Returns the image dimension ordering
    convention ('th' or 'tf').
    '''
    return _IMAGE_DIM_ORDERING

def set_image_dim_ordering(dim_ordering):
    '''Sets the value of the image dimension
    ordering convention ('th' or 'tf').
    '''
    global _IMAGE_DIM_ORDERING
    if dim_ordering not in {'tf', 'th'}:
        raise Exception('Unknown dim_ordering:', dim_ordering)
    _IMAGE_DIM_ORDERING = str(dim_ordering)

def get_uid(prefix=''):
    _UID_PREFIXES[prefix] += 1
    return _UID_PREFIXES[prefix]
<|endoftext|>


<commit_before>#!/usr/bin/env python
################################################################################
# DATE: 2017/02/28
#
# SCRIPT: run_order_regression.py
#
# VERSION: 1.3
#
# AUTHOR: Miguel A Ibarra (miguelib@ufl.edu)
#         Edited by: Matt Thoburn (mthoburn@ufl.edu)
#
# DESCRIPTION: This script takes a a wide format file and makes a run order
#              regression on it.
#
################################################################################
# Import built-in libraries
from __future__ import division
import os
import sys
import logging
import argparse
from argparse import RawDescriptionHelpFormatter

# Import add-on libraries
import matplotlib
import numpy as np
import pandas as pd
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import statsmodels.formula.api as smf
from matplotlib.backends.backend_pdf import PdfPages
from statsmodels.sandbox.regression.predstd import wls_prediction_std

# Import local libraries
from secimtools.dataManager import logger as sl
from secimtools.dataManager.flags import Flags
from secimtools.dataManager.interface import wideToDesign

# Import local plotting libraries
from secimtools.visualManager import module_lines as lines
from secimtools.visualManager import module_scatter as scatter
from secimtools.visualManager.manager_color import colorHandler
from secimtools.visualManager.manager_figure import figureHandler

def getOptions():
    """ Function to pull in arguments """
    description = """ """
    parser = argparse.ArgumentParser(description=description,
                                    formatter_class=RawDescriptionHelpFormatter)
    # Standard Input
    standard = parser.add_argument_group(title="Standard input",
                        description="Standard input for SECIM tools.")
    standard.add_argument("-i","--input", dest="input", action="store",
                        required=True, help="Input dataset in wide format.")
    standard.add_argument("-d","--design", dest="design", action="store",
                        required=True, help="Design file.")
    standard.add_argument("-id","--ID", dest="uniqID", action="store", required=True,
                        help="Name of the column with unique identifiers.")
    standard.add_argument("-g","--group", dest="group", action="store", required=True,
                        help="Group/treatment identifier in design file [Optional].")
    standard.add_argument("-o","--order", dest="order", action="store", required=True,
                        help="Name of the column on design file that contains "\
                        "run order")
    standard.add_argument("-l","--levels",dest="levels",action="store",
                        required=False, default=False, help="Different groups to"\
                        " sort by separeted by commas.")
    # Tool Output
    output = parser.add_argument_group(title="Required Output")
    output.add_argument("-f","--fig", dest="figure", action="store",
                        required=True, help="Name of PDF to save scatter plots.")
    output.add_argument("-t","--table", dest="table", action="store",
                        required=True, help="Name of table for scatter plots")
    output.add_argument("-fl","--flags", dest="flags", action="store",
                        required=True, help="Name of table flags")
    # Dev Options
    dev = parser.add_argument_group(title="Development Settings")
    dev.add_argument("-dg","--debug", dest="debug", action="store_true",
                        required=False, help="Add debugging log output.")
    # Plot Options
    plot = parser.add_argument_group(title='Plot options')
    plot.add_argument("-pal","--palette",dest="palette",action='store',
                    required=False, default="tableau", help="Name of the "\
                    "palette to use.")
    plot.add_argument("-col","--color",dest="color",action="store",
                    required=False, default="Tableau_20", help="Name of a valid"\
                    " color scheme on the selected palette")
    args = parser.parse_args()

    # Standardize paths
    args.input  = os.path.abspath(args.input)
    args.table  = os.path.abspath(args.table)
    args.flags  = os.path.abspath(args.flags)
    args.figure = os.path.abspath(args.figure)
    args.design = os.path.abspath(args.design)

    # Split levels if levels
    if args.levels:
        args.levels = args.levels.split(",")

    return (args)

def runRegression(data):
    """
    Run each column through regression and then concatenate the outputed series
    to a single dataframe.

    :Arguments:
        :type data: pandas.dataFrame
        :param data: A Pandas dataFrame with a transpose version of the original
                    data and runOrder as index.

    :Returns:
        :rtype res_df: pandas.dataFrame
        :return res_df: Results of the regression.
    """

    res = list()
    for colName in data.columns:
        column = data[colName]
        # Drop missing values and make sure everything is numeric
        clean = column.dropna().reset_index()
        clean.columns = ["run", "val"]
        try:
            clean["run"] = clean["run"].astype('float64')
        except ValueError as e:
            logger.error("Invalid runOrder value found: - {}".format(str(e)))
            sys.exit(1)

        # Fit model
        model   = smf.ols(formula="val ~ run", data=clean, missing="drop")
        results = model.fit()
        slope   = results.params["run"]
        pval    = results.pvalues["run"]
        rsq     = results.rsquared
        fitted  = results.fittedvalues

        #Returning
        out =  pd.Series(name=column.name,
                data={"name":column.name,
                "x":clean["run"],
                "y":clean["val"],
                "res":results,
                "slope":slope,
                "pval":pval,
                "rsq":rsq,
                "fitted":fitted})
        res.append(out)

    # Concatenating results
    res_df = pd.concat(res,axis=1)
    res_df = res_df.T

    # Drop rows that are missing regression results
    res_df.dropna(inplace=True)

    # Returning results of regressions
    return (res_df)

def plotSignificantROR(data, pdf, palette):
    """
    Plot a scatter plot of x vs y.

    :Arguments:

        :type row:
        :param row:

        :type pdf: PdfPages
        :param pdf: pdf object to store scatterplots

        :type des: pandas DataFrame
        :param des: design file

        :type groupName: string
        :param groupName: name of group
    """
    # Iterates over all rows in the dataframe
    # Make scatter plot if p-pvalue is less than 0.05
    for index,row in data.iterrows():
        if row["pval"] > 0.05: continue
        #plotSignificantROR(row,pdf,dat.design,args.group)

        # Get 95% CI
        prstd, lower, upper = wls_prediction_std(row["res"])

        # Sort CIs for Plotting
        toPlot = pd.DataFrame({"x": row["x"], "lower": lower, "upper": upper})
        toPlot.sort_values(by="x", inplace=True)

        # Create plot
        fh = figureHandler(proj="2d", figsize=(14,8))

        #Plot scatterplot
        scatter.scatter2D(ax=fh.ax[0],x=row["x"],  y=row["y"],colorList=palette.list_colors)

        # Plot cutoffs
        lines.drawCutoff(ax=fh.ax[0],x=row["x"],   y=row["fitted"],  c="c")
        lines.drawCutoff(ax=fh.ax[0],x=toPlot["x"],y=toPlot["lower"],c="r")
        lines.drawCutoff(ax=fh.ax[0],x=toPlot["x"],y=toPlot["upper"],c="r")

        # Formatting
        ymin, ymax = fh.ax[0].get_ylim()
        fh.formatAxis(xTitle="Run Order", yTitle="Value", ylim=(ymin,ymax*1.2),
        figTitle=u"{} Scatter plot (fitted regression line and prediction bands"\
        " included)".format(row["name"]))

        # Shrink figure
        fh.shrink()

        # Add legend to figure
        fh.makeLegend(ax=fh.ax[0], ucGroups=palette.ugColors, group=palette.combName)

        #Add text to the ax
        fh.ax[0].text(.7, .85, u"Slope= {0:.4f}\n(p-value = {1:.4f})\n"\
            "$R^2$ = {2:4f}".format(round(row["slope"],4), round(row["pval"],4),
            round(row["rsq"],4)),transform=fh.ax[0].transAxes, fontsize=12)

        # Save to PDF
        fh.addToPdf(pdf)

def main(args):
    # Checking if levels
    if args.levels and args.group:
        levels = [args.group]+args.levels
    elif args.group and not args.levels:
        levels = [args.group]
    else:
        levels = []

    #Parsing data with interface
    logger.info("Loading data with the Interface")
    dat = wideToDesign(args.input, args.design, args.uniqID, args.group,
                        runOrder=args.order, anno=args.levels, logger=logger)

    # Cleaning from missing data
    dat.dropMissing()

    # Get colors
    palette.getColors(dat.design,levels)

    # Transpose Data so compounds are columns, set the runOrder as index
    # and drop the colum with the groups from the tranposed wide.
    trans = dat.transpose()
    trans.set_index(dat.runOrder, inplace=True)
    trans.drop(dat.group, axis=1, inplace=True)

    # Run regressions
    logger.info("Running Regressions")
    ror_df = runRegression(trans)

    # Creating flags flags for pvals 0.05 and 0.1
    ror_flags = Flags(index=ror_df.index)
    ror_flags.addColumn(column="flag_feature_runOrder_pval_05",
                        mask=(ror_df["pval"]<=0.05))
    ror_flags.addColumn(column="flag_feature_runOrder_pval_01",
                        mask=(ror_df["pval"]<=0.01))

    # Plot Results
    # Open a multiple page PDF for plots
    logger.info("Plotting Results")
    with PdfPages(args.figure) as pdf:
        plotSignificantROR(ror_df, pdf, palette)

        # If not pages
        if pdf.get_pagecount() == 0:
            fig = plt.figure()
            fig.text(0.5, 0.4, "There were no features significant for plotting.", fontsize=12)
            pdf.savefig(fig)

    # Write  results and flasg to TSV files
    ror_df.to_csv(args.table, sep="\t", float_format="%.4f", index_label=args.uniqID,
                columns=["pval","rsq","slope"])
    ror_flags.df_flags.to_csv(args.flags, sep="\t", index_label=args.uniqID)

if __name__ == "__main__":
    # Command line options
    args = getOptions()

    # Setting up logger
    logger = logging.getLogger()
    if args.debug:
        sl.setLogger(logger, logLevel="debug")
        DEBUG = True
    else:
        sl.setLogger(logger)

    # Print logger info
    logger.info(u"""Importing data with following parameters:
            \tWide: {0}
            \tDesign: {1}
            \tUnique ID: {2}
            \tGroup: {3}
            \tRun Order: {4}
            \tLevels: {5}
            """ .format(args.input, args.design, args.uniqID, args.group,
                args.order,args.levels))

    # Set color palette
    palette = colorHandler(pal=args.palette, col=args.color)
    logger.info(u"Using {0} color scheme from {1} palette".format(args.color,
                args.palette))

    # Runnign code
    main(args)
<commit_msg>Fix dataset column handling in run_order_regression to make it pass the acceptance test.<commit_after>#!/usr/bin/env python
################################################################################
# AUTHORS: Miguel A Ibarra <miguelib@ufl.edu>
#          Matt Thoburn <mthoburn@ufl.edu>
#
# DESCRIPTION: Take a a wide format file and makes a run order regression on it.
################################################################################

from __future__ import division
import os
import sys
import logging
import argparse
from argparse import RawDescriptionHelpFormatter
import matplotlib
import numpy as np
import pandas as pd
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import statsmodels.formula.api as smf
from matplotlib.backends.backend_pdf import PdfPages
from statsmodels.sandbox.regression.predstd import wls_prediction_std
from secimtools.dataManager import logger as sl
from secimtools.dataManager.flags import Flags
from secimtools.dataManager.interface import wideToDesign
from secimtools.visualManager import module_lines as lines
from secimtools.visualManager import module_scatter as scatter
from secimtools.visualManager.manager_color import colorHandler
from secimtools.visualManager.manager_figure import figureHandler

def getOptions():
    """ Function to pull in arguments """
    description = """ """
    parser = argparse.ArgumentParser(description=description,
                                    formatter_class=RawDescriptionHelpFormatter)
    # Standard Input
    standard = parser.add_argument_group(title="Standard input",
                        description="Standard input for SECIM tools.")
    standard.add_argument("-i","--input", dest="input", action="store",
                        required=True, help="Input dataset in wide format.")
    standard.add_argument("-d","--design", dest="design", action="store",
                        required=True, help="Design file.")
    standard.add_argument("-id","--ID", dest="uniqID", action="store", required=True,
                        help="Name of the column with unique identifiers.")
    standard.add_argument("-g","--group", dest="group", action="store", required=True,
                        help="Group/treatment identifier in design file [Optional].")
    standard.add_argument("-o","--order", dest="order", action="store", required=True,
                        help="Name of the column on design file that contains "\
                        "run order")
    standard.add_argument("-l","--levels",dest="levels",action="store",
                        required=False, default=False, help="Different groups to"\
                        " sort by separeted by commas.")
    # Tool Output
    output = parser.add_argument_group(title="Required Output")
    output.add_argument("-f","--fig", dest="figure", action="store",
                        required=True, help="Name of PDF to save scatter plots.")
    output.add_argument("-t","--table", dest="table", action="store",
                        required=True, help="Name of table for scatter plots")
    output.add_argument("-fl","--flags", dest="flags", action="store",
                        required=True, help="Name of table flags")
    # Dev Options
    dev = parser.add_argument_group(title="Development Settings")
    dev.add_argument("-dg","--debug", dest="debug", action="store_true",
                        required=False, help="Add debugging log output.")
    # Plot Options
    plot = parser.add_argument_group(title='Plot options')
    plot.add_argument("-pal","--palette",dest="palette",action='store',
                    required=False, default="tableau", help="Name of the "\
                    "palette to use.")
    plot.add_argument("-col","--color",dest="color",action="store",
                    required=False, default="Tableau_20", help="Name of a valid"\
                    " color scheme on the selected palette")
    args = parser.parse_args()
    # Standardize paths
    args.input  = os.path.abspath(args.input)
    args.table  = os.path.abspath(args.table)
    args.flags  = os.path.abspath(args.flags)
    args.figure = os.path.abspath(args.figure)
    args.design = os.path.abspath(args.design)
    if args.levels:
        args.levels = args.levels.split(",")
    return (args)

def runRegression(data):
    """
    Run each column through regression and then concatenate the outputed series
    to a single dataframe.

    :Arguments:
        :type data: pandas.dataFrame
        :param data: A Pandas dataFrame with a transpose version of the original
                    data and runOrder as index.

    :Returns:
        :rtype res_df: pandas.dataFrame
        :return res_df: Results of the regression.
    """
    res = list()
    for colName in data.columns:
        column = data[colName]
        # Drop missing values and make sure everything is numeric
        clean = column.dropna().reset_index()
        clean.columns = ["run", "val"]
        try:
            clean["run"] = clean["run"].astype('float64')
        except ValueError as e:
            logger.error("Invalid runOrder value found: - {}".format(str(e)))
            sys.exit(1)

        # Fit model
        model   = smf.ols(formula="val ~ run", data=clean, missing="drop")
        results = model.fit()
        slope   = results.params["run"]
        pval    = results.pvalues["run"]
        rsq     = results.rsquared
        fitted  = results.fittedvalues

        #Returning
        out =  pd.Series(name=column.name,
                data={"name":column.name,
                "x":clean["run"],
                "y":clean["val"],
                "res":results,
                "slope":slope,
                "pval":pval,
                "rsq":rsq,
                "fitted":fitted})
        res.append(out)

    # Concatenating results
    res_df = pd.concat(res,axis=1)
    res_df = res_df.T

    # Drop rows that are missing regression results
    res_df.dropna(inplace=True)

    # Returning results of regressions
    return (res_df)

def plotSignificantROR(data, pdf, palette):
    """
    Plot a scatter plot of x vs y.

    :Arguments:

        :type row:
        :param row:

        :type pdf: PdfPages
        :param pdf: pdf object to store scatterplots

        :type des: pandas DataFrame
        :param des: design file

        :type groupName: string
        :param groupName: name of group
    """
    # Iterates over all rows in the dataframe
    # Make scatter plot if p-pvalue is less than 0.05
    for index,row in data.iterrows():
        if row["pval"] > 0.05: continue
        #plotSignificantROR(row,pdf,dat.design,args.group)

        # Get 95% CI
        prstd, lower, upper = wls_prediction_std(row["res"])

        # Sort CIs for Plotting
        toPlot = pd.DataFrame({"x": row["x"], "lower": lower, "upper": upper})
        toPlot.sort_values(by="x", inplace=True)

        # Create plot
        fh = figureHandler(proj="2d", figsize=(14,8))

        #Plot scatterplot
        scatter.scatter2D(ax=fh.ax[0],x=row["x"],  y=row["y"],colorList=palette.list_colors)

        # Plot cutoffs
        lines.drawCutoff(ax=fh.ax[0],x=row["x"],   y=row["fitted"],  c="c")
        lines.drawCutoff(ax=fh.ax[0],x=toPlot["x"],y=toPlot["lower"],c="r")
        lines.drawCutoff(ax=fh.ax[0],x=toPlot["x"],y=toPlot["upper"],c="r")

        # Formatting
        ymin, ymax = fh.ax[0].get_ylim()
        fh.formatAxis(xTitle="Run Order", yTitle="Value", ylim=(ymin,ymax*1.2),
        figTitle=u"{} Scatter plot (fitted regression line and prediction bands"\
        " included)".format(row["name"]))

        # Shrink figure
        fh.shrink()

        # Add legend to figure
        fh.makeLegend(ax=fh.ax[0], ucGroups=palette.ugColors, group=palette.combName)

        #Add text to the ax
        fh.ax[0].text(.7, .85, u"Slope= {0:.4f}\n(p-value = {1:.4f})\n"\
            "$R^2$ = {2:4f}".format(round(row["slope"],4), round(row["pval"],4),
            round(row["rsq"],4)),transform=fh.ax[0].transAxes, fontsize=12)

        # Save to PDF
        fh.addToPdf(pdf)

def main(args):
    """Perform run order regression analysis"""

    if args.levels and args.group:
        levels = [args.group]+args.levels
    elif args.group and not args.levels:
        levels = [args.group]
    else:
        levels = []

    logger.info("Loading data with the Interface")
    dat = wideToDesign(args.input, args.design, args.uniqID, args.group,
                        runOrder=args.order, anno=args.levels, logger=logger)

    # Remove samples with missing data
    dat.dropMissing()
    # Get remaining Sample IDs for dataframe filtering
    sample_ids = dat.wide.index.tolist()
    # Set color palette and get colors
    palette = colorHandler(pal=args.palette, col=args.color)
    logger.info(u"Using {0} color scheme from {1} palette".format(args.color, args.palette))
    palette.getColors(dat.design,levels)

    # Transpose Data so compounds are columns, set the runOrder as index
    # and drop the colum with the groups from the tranposed wide.
    trans = dat.transpose()
    trans.set_index(dat.runOrder, inplace=True)
    trans.drop(dat.group, axis=1, inplace=True)
    trans = trans.loc[:,sample_ids]
    # Run regressions
    logger.info("Running Regressions")
    ror_df = runRegression(trans)

    # Creating flags flags for pvals 0.05 and 0.1
    ror_flags = Flags(index=ror_df.index)
    ror_flags.addColumn(column="flag_feature_runOrder_pval_05",
                        mask=(ror_df["pval"]<=0.05))
    ror_flags.addColumn(column="flag_feature_runOrder_pval_01",
                        mask=(ror_df["pval"]<=0.01))

    # Plot Results
    # Open a multiple page PDF for plots
    logger.info("Plotting Results")
    with PdfPages(args.figure) as pdf:
        plotSignificantROR(ror_df, pdf, palette)

        # If not pages
        if pdf.get_pagecount() == 0:
            fig = plt.figure()
            fig.text(0.5, 0.4, "There were no features significant for plotting.", fontsize=12)
            pdf.savefig(fig)

    # Write  results and flasg to TSV files
    ror_df.to_csv(args.table, sep="\t", float_format="%.4f", index_label=args.uniqID,
                columns=["pval","rsq","slope"])
    ror_flags.df_flags.to_csv(args.flags, sep="\t", index_label=args.uniqID)

if __name__ == "__main__":
    """Run if executed on the command line"""
    args = getOptions()
    logger = logging.getLogger()
    if args.debug:
        sl.setLogger(logger, logLevel="debug")
        DEBUG = True
    else:
        sl.setLogger(logger)
    # Print logger info
    logger.info(u"""Importing data with following parameters:
            \tWide: {0}
            \tDesign: {1}
            \tUnique ID: {2}
            \tGroup: {3}
            \tRun Order: {4}
            \tLevels: {5}
            """ .format(args.input, args.design, args.uniqID, args.group,
                args.order,args.levels))

    main(args)

<|endoftext|>


<commit_before># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Built-in linear model classes."""

import tensorflow.compat.v2 as tf
from keras import activations
from keras import initializers
from keras import regularizers
from keras.engine import base_layer
from keras.engine import input_spec
from keras.engine import training
from keras.layers import core
from tensorflow.python.util import deprecation  # pylint: disable=g-direct-tensorflow-import
from tensorflow.python.util.tf_export import keras_export

@keras_export(
    'keras.experimental.LinearModel',
    v1=['keras.experimental.LinearModel', 'keras.models.LinearModel'])
@deprecation.deprecated_endpoints('keras.experimental.LinearModel')
class LinearModel(training.Model):
  r"""Linear Model for regression and classification problems.

  This model approximates the following function:
  $$y = \beta + \sum_{i=1}^{N} w_{i} * x_{i}$$
  where $$\beta$$ is the bias and $$w_{i}$$ is the weight for each feature.

  Example:

  ```python
  model = LinearModel()
  model.compile(optimizer='sgd', loss='mse')
  model.fit(x, y, epochs=epochs)
  ```

  This model accepts sparse float inputs as well:

  Example:
  ```python
  model = LinearModel()
  opt = tf.keras.optimizers.Adam()
  loss_fn = tf.keras.losses.MeanSquaredError()
  with tf.GradientTape() as tape:
    output = model(sparse_input)
    loss = tf.reduce_mean(loss_fn(target, output))
  grads = tape.gradient(loss, model.weights)
  opt.apply_gradients(zip(grads, model.weights))
  ```

  """

  def __init__(self,
               units=1,
               activation=None,
               use_bias=True,
               kernel_initializer='zeros',
               bias_initializer='zeros',
               kernel_regularizer=None,
               bias_regularizer=None,
               **kwargs):
    """Create a Linear Model.

    Args:
      units: Positive integer, output dimension without the batch size.
      activation: Activation function to use.
        If you don't specify anything, no activation is applied.
      use_bias: whether to calculate the bias/intercept for this model. If set
        to False, no bias/intercept will be used in calculations, e.g., the data
        is already centered.
      kernel_initializer: Initializer for the `kernel` weights matrices.
      bias_initializer: Initializer for the bias vector.
      kernel_regularizer: regularizer for kernel vectors.
      bias_regularizer: regularizer for bias vector.
      **kwargs: The keyword arguments that are passed on to BaseLayer.__init__.
    """

    self.units = units
    self.activation = activations.get(activation)
    self.use_bias = use_bias
    self.kernel_initializer = initializers.get(kernel_initializer)
    self.bias_initializer = initializers.get(bias_initializer)
    self.kernel_regularizer = regularizers.get(kernel_regularizer)
    self.bias_regularizer = regularizers.get(bias_regularizer)
    super(LinearModel, self).__init__(**kwargs)
    base_layer.keras_premade_model_gauge.get_cell('Linear').set(True)

  def build(self, input_shape):
    if isinstance(input_shape, dict):
      names = sorted(list(input_shape.keys()))
      self.input_specs = []
      self.dense_layers = []
      for name in names:
        shape = input_shape[name]
        layer = core.Dense(
            units=self.units,
            use_bias=False,
            kernel_initializer=self.kernel_initializer,
            kernel_regularizer=self.kernel_regularizer,
            name=name)
        layer.build(shape)
        self.input_specs.append(
            input_spec.InputSpec(shape=shape, name=name))
        self.dense_layers.append(layer)
    elif isinstance(input_shape, (tuple, list)) and all(
        isinstance(shape, tf.TensorShape) for shape in input_shape):
      self.dense_layers = []
      for shape in input_shape:
        layer = core.Dense(
            units=self.units,
            use_bias=False,
            kernel_initializer=self.kernel_initializer,
            kernel_regularizer=self.kernel_regularizer)
        layer.build(shape)
        self.dense_layers.append(layer)
    else:
      # input_shape can be a single TensorShape or a tuple of ints.
      layer = core.Dense(
          units=self.units,
          use_bias=False,
          kernel_initializer=self.kernel_initializer,
          kernel_regularizer=self.kernel_regularizer)
      layer.build(input_shape)
      self.dense_layers = [layer]

    if self.use_bias:
      self.bias = self.add_weight(
          'bias',
          shape=self.units,
          initializer=self.bias_initializer,
          regularizer=self.bias_regularizer,
          dtype=self.dtype,
          trainable=True)
    else:
      self.bias = None
    self.built = True

  def call(self, inputs):
    result = None
    if isinstance(inputs, dict):
      names = [layer.name for layer in self.dense_layers]
      different_keys = set(names) - set(inputs.keys())
      if different_keys:
        raise ValueError(
            'The input dictionary does not match '
            'the structure expected by the model.'
            '\n\tExpected keys: {}'
            '\n\tReceived keys: {}'
            '\n\tMissing keys: {}'.format(set(names), set(inputs.keys()),
                                          different_keys))
      inputs = [inputs[name] for name in names]
      for inp, layer in zip(inputs, self.dense_layers):
        output = layer(inp)
        if result is None:
          result = output
        else:
          result += output
    elif isinstance(inputs, (tuple, list)):
      for inp, layer in zip(inputs, self.dense_layers):
        output = layer(inp)
        if result is None:
          result = output
        else:
          result += output
    else:
      result = self.dense_layers[0](inputs)

    if self.use_bias:
      result = tf.nn.bias_add(result, self.bias)
    if self.activation is not None:
      return self.activation(result)  # pylint: disable=not-callable
    return result

  def get_config(self):
    config = {
        'units': self.units,
        'activation': activations.serialize(self.activation),
        'use_bias': self.use_bias,
        'kernel_initializer': initializers.serialize(self.kernel_initializer),
        'bias_initializer': initializers.serialize(self.bias_initializer),
        'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
        'bias_regularizer': regularizers.serialize(self.bias_regularizer),
    }
    base_config = base_layer.Layer.get_config(self)
    return dict(list(base_config.items()) + list(config.items()))

  @classmethod
  def from_config(cls, config, custom_objects=None):
    del custom_objects
    return cls(**config)
<commit_msg>Update `keras/premade/linear.py` error message to use f-strings instead of `.format()`<commit_after># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Built-in linear model classes."""

import tensorflow.compat.v2 as tf
from keras import activations
from keras import initializers
from keras import regularizers
from keras.engine import base_layer
from keras.engine import input_spec
from keras.engine import training
from keras.layers import core
from tensorflow.python.util import deprecation  # pylint: disable=g-direct-tensorflow-import
from tensorflow.python.util.tf_export import keras_export

@keras_export(
    'keras.experimental.LinearModel',
    v1=['keras.experimental.LinearModel', 'keras.models.LinearModel'])
@deprecation.deprecated_endpoints('keras.experimental.LinearModel')
class LinearModel(training.Model):
  r"""Linear Model for regression and classification problems.

  This model approximates the following function:
  $$y = \beta + \sum_{i=1}^{N} w_{i} * x_{i}$$
  where $$\beta$$ is the bias and $$w_{i}$$ is the weight for each feature.

  Example:

  ```python
  model = LinearModel()
  model.compile(optimizer='sgd', loss='mse')
  model.fit(x, y, epochs=epochs)
  ```

  This model accepts sparse float inputs as well:

  Example:
  ```python
  model = LinearModel()
  opt = tf.keras.optimizers.Adam()
  loss_fn = tf.keras.losses.MeanSquaredError()
  with tf.GradientTape() as tape:
    output = model(sparse_input)
    loss = tf.reduce_mean(loss_fn(target, output))
  grads = tape.gradient(loss, model.weights)
  opt.apply_gradients(zip(grads, model.weights))
  ```

  """

  def __init__(self,
               units=1,
               activation=None,
               use_bias=True,
               kernel_initializer='zeros',
               bias_initializer='zeros',
               kernel_regularizer=None,
               bias_regularizer=None,
               **kwargs):
    """Create a Linear Model.

    Args:
      units: Positive integer, output dimension without the batch size.
      activation: Activation function to use.
        If you don't specify anything, no activation is applied.
      use_bias: whether to calculate the bias/intercept for this model. If set
        to False, no bias/intercept will be used in calculations, e.g., the data
        is already centered.
      kernel_initializer: Initializer for the `kernel` weights matrices.
      bias_initializer: Initializer for the bias vector.
      kernel_regularizer: regularizer for kernel vectors.
      bias_regularizer: regularizer for bias vector.
      **kwargs: The keyword arguments that are passed on to BaseLayer.__init__.
    """

    self.units = units
    self.activation = activations.get(activation)
    self.use_bias = use_bias
    self.kernel_initializer = initializers.get(kernel_initializer)
    self.bias_initializer = initializers.get(bias_initializer)
    self.kernel_regularizer = regularizers.get(kernel_regularizer)
    self.bias_regularizer = regularizers.get(bias_regularizer)
    super(LinearModel, self).__init__(**kwargs)
    base_layer.keras_premade_model_gauge.get_cell('Linear').set(True)

  def build(self, input_shape):
    if isinstance(input_shape, dict):
      names = sorted(list(input_shape.keys()))
      self.input_specs = []
      self.dense_layers = []
      for name in names:
        shape = input_shape[name]
        layer = core.Dense(
            units=self.units,
            use_bias=False,
            kernel_initializer=self.kernel_initializer,
            kernel_regularizer=self.kernel_regularizer,
            name=name)
        layer.build(shape)
        self.input_specs.append(
            input_spec.InputSpec(shape=shape, name=name))
        self.dense_layers.append(layer)
    elif isinstance(input_shape, (tuple, list)) and all(
        isinstance(shape, tf.TensorShape) for shape in input_shape):
      self.dense_layers = []
      for shape in input_shape:
        layer = core.Dense(
            units=self.units,
            use_bias=False,
            kernel_initializer=self.kernel_initializer,
            kernel_regularizer=self.kernel_regularizer)
        layer.build(shape)
        self.dense_layers.append(layer)
    else:
      # input_shape can be a single TensorShape or a tuple of ints.
      layer = core.Dense(
          units=self.units,
          use_bias=False,
          kernel_initializer=self.kernel_initializer,
          kernel_regularizer=self.kernel_regularizer)
      layer.build(input_shape)
      self.dense_layers = [layer]

    if self.use_bias:
      self.bias = self.add_weight(
          'bias',
          shape=self.units,
          initializer=self.bias_initializer,
          regularizer=self.bias_regularizer,
          dtype=self.dtype,
          trainable=True)
    else:
      self.bias = None
    self.built = True

  def call(self, inputs):
    result = None
    if isinstance(inputs, dict):
      names = [layer.name for layer in self.dense_layers]
      different_keys = set(names) - set(inputs.keys())
      if different_keys:
        raise ValueError(
            'The `inputs` dictionary does not match '
            'the structure expected by the model.'
            f'\n\tExpected keys: {set(names)}'
            f'\n\tReceived keys: {set(inputs.keys())}'
            f'\n\tMissing keys: {different_keys}')
      inputs = [inputs[name] for name in names]
      for inp, layer in zip(inputs, self.dense_layers):
        output = layer(inp)
        if result is None:
          result = output
        else:
          result += output
    elif isinstance(inputs, (tuple, list)):
      for inp, layer in zip(inputs, self.dense_layers):
        output = layer(inp)
        if result is None:
          result = output
        else:
          result += output
    else:
      result = self.dense_layers[0](inputs)

    if self.use_bias:
      result = tf.nn.bias_add(result, self.bias)
    if self.activation is not None:
      return self.activation(result)  # pylint: disable=not-callable
    return result

  def get_config(self):
    config = {
        'units': self.units,
        'activation': activations.serialize(self.activation),
        'use_bias': self.use_bias,
        'kernel_initializer': initializers.serialize(self.kernel_initializer),
        'bias_initializer': initializers.serialize(self.bias_initializer),
        'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
        'bias_regularizer': regularizers.serialize(self.bias_regularizer),
    }
    base_config = base_layer.Layer.get_config(self)
    return dict(list(base_config.items()) + list(config.items()))

  @classmethod
  def from_config(cls, config, custom_objects=None):
    del custom_objects
    return cls(**config)
<|endoftext|>


<commit_before>"""
Created on 1 Mar 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

example JSON:
{"calibrated-on": "2019-02-02T11:34:16Z", "offset": 50, "env": {"hmd": 66.0, "tmp": 11.0, "pA": 99.0}}
"""

from collections import OrderedDict

from scs_core.data.datum import Datum
from scs_core.data.localized_datetime import LocalizedDatetime
from scs_core.data.json import JSONable

# --------------------------------------------------------------------------------------------------------------------

class SensorBaseline(JSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct_from_jdict(cls, jdict):
        if not jdict:
            return SensorBaseline(None, 0, None)

        if 'calibrated_on' in jdict:                            # TODO: deprecated
            date = Datum.date(jdict.get('calibrated_on'))
            calibrated_on = LocalizedDatetime.construct_from_date(date)

        else:
            calibrated_on = Datum.datetime(jdict.get('calibrated-on'))

        offset = jdict.get('offset')
        environment = BaselineEnvironment.construct_from_jdict(jdict.get('env'))

        return SensorBaseline(calibrated_on, offset, environment)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, calibrated_on, offset, environment):
        """
        Constructor
        """
        self.__calibrated_on = calibrated_on            # LocalizedDatetime

        self.__offset = Datum.int(offset)               # int                       ppb
        self.__environment = environment                # BaselineEnvironment

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        jdict = OrderedDict()

        jdict['calibrated-on'] = None if self.calibrated_on is None else self.calibrated_on.as_iso8601(False)

        jdict['offset'] = self.offset
        jdict['env'] = self.environment

        return jdict

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def calibrated_on(self):
        return self.__calibrated_on

    @property
    def offset(self):
        return self.__offset

    @property
    def environment(self):
        return self.__environment

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "SensorBaseline:{calibrated_on:%s, offset:%s, environment:%s}" % \
               (self.calibrated_on, self.offset, self.environment)

# --------------------------------------------------------------------------------------------------------------------

class BaselineEnvironment(JSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct_from_jdict(cls, jdict):
        if not jdict:
            return None

        humid = jdict.get('hmd')
        temp = jdict.get('tmp')
        press = jdict.get('pA')

        return BaselineEnvironment(humid, temp, press)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, humid, temp, press):
        """
        Constructor
        """
        self.__humid = Datum.float(humid, 1)            # float                 %
        self.__temp = Datum.float(temp, 1)              # float                 °C
        self.__press = Datum.float(press, 1)            # float                 kPa

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        jdict = OrderedDict()

        jdict['hmd'] = self.humid
        jdict['tmp'] = self.temp
        jdict['pA'] = self.press

        return jdict

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def humid(self):
        return self.__humid

    @property
    def temp(self):
        return self.__temp

    @property
    def press(self):
        return self.__press

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "BaselineEnvironment:{humid:%s, temp:%s, press:%s}" % (self.humid, self.temp, self.press)
<commit_msg>Added environmental parameters to AFEBaseline.<commit_after>"""
Created on 1 Mar 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

example JSON:
{"calibrated-on": "2019-02-02T11:34:16Z", "offset": 50, "env": {"hmd": 66.0, "tmp": 11.0, "pA": 99.0}}
"""

from collections import OrderedDict

from scs_core.data.datum import Datum
from scs_core.data.json import JSONable
from scs_core.data.localized_datetime import LocalizedDatetime

# --------------------------------------------------------------------------------------------------------------------

class SensorBaseline(JSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct_from_jdict(cls, jdict):
        if not jdict:
            return SensorBaseline(None, 0, None)

        if 'calibrated_on' in jdict:                            # TODO: deprecated
            date = Datum.date(jdict.get('calibrated_on'))
            calibrated_on = LocalizedDatetime.construct_from_date(date)

        else:
            calibrated_on = Datum.datetime(jdict.get('calibrated-on'))

        offset = jdict.get('offset')
        environment = BaselineEnvironment.construct_from_jdict(jdict.get('env'))

        return SensorBaseline(calibrated_on, offset, environment)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, calibrated_on, offset, environment):
        """
        Constructor
        """
        self.__calibrated_on = calibrated_on            # LocalizedDatetime

        self.__offset = Datum.int(offset)               # int                       ppb
        self.__environment = environment                # BaselineEnvironment

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        jdict = OrderedDict()

        jdict['calibrated-on'] = None if self.calibrated_on is None else self.calibrated_on.as_iso8601(False)

        jdict['offset'] = self.offset
        jdict['env'] = self.environment

        return jdict

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def calibrated_on(self):
        return self.__calibrated_on

    @property
    def offset(self):
        return self.__offset

    @property
    def environment(self):
        return self.__environment

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "SensorBaseline:{calibrated_on:%s, offset:%s, environment:%s}" % \
               (self.calibrated_on, self.offset, self.environment)

# --------------------------------------------------------------------------------------------------------------------

class BaselineEnvironment(JSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct_from_jdict(cls, jdict):
        if not jdict:
            return None

        humid = jdict.get('hmd')
        temp = jdict.get('tmp')
        press = jdict.get('pA')

        return BaselineEnvironment(humid, temp, press)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, humid, temp, press):
        """
        Constructor
        """
        self.__humid = Datum.float(humid, 1)            # float                 %
        self.__temp = Datum.float(temp, 1)              # float                 °C
        self.__press = Datum.float(press, 1)            # float                 kPa

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        jdict = OrderedDict()

        jdict['hmd'] = self.humid
        jdict['tmp'] = self.temp
        jdict['pA'] = self.press

        return jdict

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def humid(self):
        return self.__humid

    @property
    def temp(self):
        return self.__temp

    @property
    def press(self):
        return self.__press

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "BaselineEnvironment:{humid:%s, temp:%s, press:%s}" % (self.humid, self.temp, self.press)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import atexit
import math
import sys

import pygtk
pygtk.require('2.0')
import gtk
import keybinder

import windows

HOTKEY = '<Alt>R'
CANVAS_SCALE = 0.2
BOXES_PER_SIDE = 8

class RationApp:
    """
    An application to quickly resize windows to fill discrete portions of the
    screen in GTK-based Linux distributions.
    """
    mouse_down = None
    canvas_width = 0
    canvas_height = 0

    def __init__(self):
        """
        Setup the window and canvas.
        """
        self.window = gtk.Window()
        self.window.set_decorated(False)
        self.window.set_position(gtk.WIN_POS_CENTER_ALWAYS)
        self.window.set_resizable(False)
        self.window.set_keep_above(True)

        self.canvas = gtk.DrawingArea()
        self.canvas.set_events(
            gtk.gdk.POINTER_MOTION_MASK | gtk.gdk.POINTER_MOTION_HINT_MASK |
            gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK)
        self.window.add(self.canvas)

        self.window.connect('destroy', gtk.main_quit)
        self.window.connect('destroy-event', gtk.main_quit)
        self.canvas.connect('configure-event', self.canvas_configure)
        self.canvas.connect('expose-event', self.canvas_expose)
        self.canvas.connect('motion-notify-event', self.canvas_motion)
        self.canvas.connect('button-press-event', self.canvas_button_press)
        self.canvas.connect('button-release-event', self.canvas_button_release)

        width, height = windows.get_screen_resolution()
        width = math.floor(width * CANVAS_SCALE / BOXES_PER_SIDE) * BOXES_PER_SIDE + 1
        height = math.floor(height * CANVAS_SCALE / BOXES_PER_SIDE) * BOXES_PER_SIDE + 1

        self.setup_status_icon()

        self.window.set_size_request(int(width), int(height))
        self.window.show_all()

        self.bind_hotkey()

    def setup_status_icon(self):
        """
        Setup the status icon that functions as a main menu.
        """
        self.menu = gtk.Menu()
        quit_item = gtk.ImageMenuItem(gtk.STOCK_QUIT)
        quit_item.connect('activate', self.menu_quit)
        self.menu.append(quit_item)
        self.menu.show_all()

        self.status_icon = gtk.StatusIcon()
        self.status_icon.set_from_stock(gtk.STOCK_FIND)
        self.status_icon.set_tooltip('Ration')
        self.status_icon.set_visible(True)

        self.status_icon.connect('activate', self.status_activate)
        self.status_icon.connect('popup-menu', self.status_popup_menu)

    def status_activate(self, status_icon):
        """
        Toggle the visiblity of the app window.
        """
        self.hotkey()

    def status_popup_menu(self, status_icon, button, time):
        """
        Show the application menu.
        """
        self.menu.popup(None, None, None, button, time)

    def menu_quit(self, menu_item):
        """
        Exit the application from menu.
        """
        gtk.main_quit()

    def canvas_configure(self, widget, event):
        """
        Configure the rendering back-buffer.
        """
        x, y, self.canvas_width, self.canvas_height = widget.get_allocation()

        self.buffer_pixmap = gtk.gdk.Pixmap(widget.window, self.canvas_width, self.canvas_height)
        self.clear_buffer()

        return True

    def canvas_expose(self, widget, event):
        """
        Render the back-buffer to the canvas.
        """
        x, y, width, height = event.area

        widget.window.draw_drawable(
            self.window.get_style().fg_gc[gtk.STATE_NORMAL],
            self.buffer_pixmap,
            x, y, x, y, width, height)

        return False

    def canvas_motion(self, widget, event):
        """
        Redraw the canvas with the latest selection.
        """
        if self.mouse_down:
            self.update(event)

        return True

    def canvas_button_press(self, widget, event):
        """
        Record mouse coordinates for drawing the selection box.
        """
        self.mouse_down = (event.x, event.y)

        return True

    def canvas_button_release(self, widget, event):
        """
        Redraw the canvas with the latest selection.
        """
        self.update(event, selection=False)

        self.mouse_down = None

        window_id, window_name = windows.select_window()

        if window_id != hex(self.window.window.xid)[:-1] and \
            'Edge Panel' not in window_name:
            # If all boxes selected then maximize instead of resizing
            if self.selected_boxes == (0, 0, BOXES_PER_SIDE, BOXES_PER_SIDE):
                windows.maximize_window(window_id)

            windows.resize_window(window_id, *self.new_window_size)

        self.clear_buffer()
        self.draw_grid()
        self.blit_buffer()

        return True

    def update(self, event, selection=True):
        """
        Updates selection coordinates and redraws the GUI.
        """
        self.compute_selection_rectangle(event)
        self.compute_selected_boxes()
        self.compute_new_window_size()
        self.clear_buffer()
        self.draw_selected_boxes()
        self.draw_grid()
        if selection: self.draw_selection()
        self.blit_buffer()

    def compute_selection_rectangle(self, event):
        """
        Compute the shape of the selection, taking into account negative selection coordinates.

        (i.e. lower-right to upper-left)
        """
        x = self.mouse_down[0]
        y = self.mouse_down[1]
        width = event.x - x
        height = event.y - y

        if width < 0:
            width = -width
            x -= width

        if height < 0:
            height = -height
            y -= height

        self.selection = (int(x), int(y), int(x + width), int(y + height))

    def compute_selected_boxes(self):
        """
        Use mouse coordinates to determine which boxes have been selected.
        """
        self.selected_boxes = (int(math.floor(self.selection[0] / (self.canvas_width / BOXES_PER_SIDE))),
                               int(math.floor(self.selection[1] / (self.canvas_height / BOXES_PER_SIDE))),
                               int(math.floor(self.selection[2] / (self.canvas_width / BOXES_PER_SIDE)) + 1),
                               int(math.floor(self.selection[3] / (self.canvas_height / BOXES_PER_SIDE)) + 1))

    def compute_new_window_size(self):
        """
        Translate the selection size to the actual screen size a window should be resized too.
        """
        self.new_window_size = (self.selected_boxes[0] * self.canvas_width / BOXES_PER_SIDE / CANVAS_SCALE,
                                self.selected_boxes[1] * self.canvas_height / BOXES_PER_SIDE / CANVAS_SCALE,
                                (self.selected_boxes[2] - self.selected_boxes[0]) * self.canvas_width / BOXES_PER_SIDE / CANVAS_SCALE,
                                (self.selected_boxes[3] - self.selected_boxes[1]) * self.canvas_height / BOXES_PER_SIDE / CANVAS_SCALE)

    def clear_buffer(self):
        """
        Clear the back-buffer.
        """
        self.buffer_pixmap.draw_rectangle(self.window.get_style().white_gc, True, 0, 0, self.canvas_width, self.canvas_height)
        self.draw_grid()

    def draw_grid(self):
        """
        Draw the selection grid onto the back-buffer.
        """
        for i in range(0, BOXES_PER_SIDE + 1):
            x = (self.canvas_width / BOXES_PER_SIDE) * i
            self.buffer_pixmap.draw_line(self.window.get_style().black_gc, x, 0, x, self.canvas_height)

        for j in range(0, BOXES_PER_SIDE + 1):
            y = (self.canvas_height / BOXES_PER_SIDE) * j
            self.buffer_pixmap.draw_line(self.window.get_style().black_gc, 0, y, self.canvas_width, y)

    def draw_selection(self):
        """
        Draw the current selection box onto the back-buffer.
        """
        color = self.buffer_pixmap.get_colormap().alloc_color("#FF9999")

        style = self.window.get_style()
        style.fg[gtk.STATE_NORMAL] = color

        self.buffer_pixmap.draw_rectangle(
            style.fg_gc[gtk.STATE_NORMAL],
            False,
            self.selection[0],
            self.selection[1],
            self.selection[2] - self.selection[0],
            self.selection[3] - self.selection[1])

    def draw_selected_boxes(self):
        """
        Draw the selected boxes in a different color.
        """
        x1 = self.selected_boxes[0] * self.canvas_width / BOXES_PER_SIDE
        y1 = self.selected_boxes[1] * self.canvas_height / BOXES_PER_SIDE
        x2 = self.selected_boxes[2] * self.canvas_width / BOXES_PER_SIDE
        y2 = self.selected_boxes[3] * self.canvas_height / BOXES_PER_SIDE

        self.buffer_pixmap.draw_rectangle(self.window.get_style().black_gc, True, int(x1), int(y1), int(x2 - x1), int(y2 - y1))

    def blit_buffer(self):
        """
        Copy the back-buffer to the visible canvas.
        """
        self.canvas.window.draw_drawable(
            self.window.get_style().fg_gc[gtk.STATE_NORMAL],
            self.buffer_pixmap,
            0, 0, 0, 0, self.canvas_width, self.canvas_height)

    def hotkey(self):
        """
        Toggle the visibility of the window.
        """
        if self.window.get_visible():
            self.window.hide()
        else:
            self.window.show()

    def bind_hotkey(self):
        """
        Bind the toggle hotkey.
        """
        keybinder.bind(HOTKEY, self.hotkey)

    def unbind_hotkey(self):
        """
        Unbind the toggle hotkey.
        """
        keybinder.unbind(HOTKEY)

if __name__ == '__main__':
    app = RationApp()
    atexit.register(app.unbind_hotkey)
    gtk.main()

<commit_msg>Colored selected boxes. Hid selection rectangle.<commit_after>#!/usr/bin/env python

import atexit
import math
import sys

import pygtk
pygtk.require('2.0')
import gtk
import keybinder

import windows

HOTKEY = '<Alt>R'
CANVAS_SCALE = 0.2
BOXES_PER_SIDE = 8

class RationApp:
    """
    An application to quickly resize windows to fill discrete portions of the
    screen in GTK-based Linux distributions.
    """
    mouse_down = None
    canvas_width = 0
    canvas_height = 0

    def __init__(self):
        """
        Setup the window and canvas.
        """
        self.window = gtk.Window()
        self.window.set_decorated(False)
        self.window.set_position(gtk.WIN_POS_CENTER_ALWAYS)
        self.window.set_resizable(False)
        self.window.set_keep_above(True)

        self.canvas = gtk.DrawingArea()
        self.canvas.set_events(
            gtk.gdk.POINTER_MOTION_MASK | gtk.gdk.POINTER_MOTION_HINT_MASK |
            gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK)
        self.window.add(self.canvas)

        self.window.connect('destroy', gtk.main_quit)
        self.window.connect('destroy-event', gtk.main_quit)
        self.canvas.connect('configure-event', self.canvas_configure)
        self.canvas.connect('expose-event', self.canvas_expose)
        self.canvas.connect('motion-notify-event', self.canvas_motion)
        self.canvas.connect('button-press-event', self.canvas_button_press)
        self.canvas.connect('button-release-event', self.canvas_button_release)

        width, height = windows.get_screen_resolution()
        width = math.floor(width * CANVAS_SCALE / BOXES_PER_SIDE) * BOXES_PER_SIDE + 1
        height = math.floor(height * CANVAS_SCALE / BOXES_PER_SIDE) * BOXES_PER_SIDE + 1

        self.setup_status_icon()

        self.window.set_size_request(int(width), int(height))
        self.window.show_all()

        self.bind_hotkey()

    def setup_status_icon(self):
        """
        Setup the status icon that functions as a main menu.
        """
        self.menu = gtk.Menu()
        quit_item = gtk.ImageMenuItem(gtk.STOCK_QUIT)
        quit_item.connect('activate', self.menu_quit)
        self.menu.append(quit_item)
        self.menu.show_all()

        self.status_icon = gtk.StatusIcon()
        self.status_icon.set_from_stock(gtk.STOCK_FIND)
        self.status_icon.set_tooltip('Ration')
        self.status_icon.set_visible(True)

        self.status_icon.connect('activate', self.status_activate)
        self.status_icon.connect('popup-menu', self.status_popup_menu)

    def status_activate(self, status_icon):
        """
        Toggle the visiblity of the app window.
        """
        self.hotkey()

    def status_popup_menu(self, status_icon, button, time):
        """
        Show the application menu.
        """
        self.menu.popup(None, None, None, button, time)

    def menu_quit(self, menu_item):
        """
        Exit the application from menu.
        """
        gtk.main_quit()

    def canvas_configure(self, widget, event):
        """
        Configure the rendering back-buffer.
        """
        x, y, self.canvas_width, self.canvas_height = widget.get_allocation()

        self.buffer_pixmap = gtk.gdk.Pixmap(widget.window, self.canvas_width, self.canvas_height)
        self.clear_buffer()

        return True

    def canvas_expose(self, widget, event):
        """
        Render the back-buffer to the canvas.
        """
        x, y, width, height = event.area

        widget.window.draw_drawable(
            self.window.get_style().fg_gc[gtk.STATE_NORMAL],
            self.buffer_pixmap,
            x, y, x, y, width, height)

        return False

    def canvas_motion(self, widget, event):
        """
        Redraw the canvas with the latest selection.
        """
        if self.mouse_down:
            self.update(event)

        return True

    def canvas_button_press(self, widget, event):
        """
        Record mouse coordinates for drawing the selection box.
        """
        self.mouse_down = (event.x, event.y)

        return True

    def canvas_button_release(self, widget, event):
        """
        Redraw the canvas with the latest selection.
        """
        self.update(event, selection=False)

        self.mouse_down = None

        window_id, window_name = windows.select_window()

        if window_id != hex(self.window.window.xid)[:-1] and \
            'Edge Panel' not in window_name:
            # If all boxes selected then maximize instead of resizing
            if self.selected_boxes == (0, 0, BOXES_PER_SIDE, BOXES_PER_SIDE):
                windows.maximize_window(window_id)

            windows.resize_window(window_id, *self.new_window_size)

        self.clear_buffer()
        self.draw_grid()
        self.blit_buffer()

        return True

    def update(self, event, selection=True):
        """
        Updates selection coordinates and redraws the GUI.
        """
        self.compute_selection_rectangle(event)
        self.compute_selected_boxes()
        self.compute_new_window_size()
        self.clear_buffer()
        self.draw_selected_boxes()
        self.draw_grid()
#        if selection: self.draw_selection()
        self.blit_buffer()

    def compute_selection_rectangle(self, event):
        """
        Compute the shape of the selection, taking into account negative selection coordinates.

        (i.e. lower-right to upper-left)
        """
        x = self.mouse_down[0]
        y = self.mouse_down[1]
        width = event.x - x
        height = event.y - y

        if width < 0:
            width = -width
            x -= width

        if height < 0:
            height = -height
            y -= height

        self.selection = (int(x), int(y), int(x + width), int(y + height))

    def compute_selected_boxes(self):
        """
        Use mouse coordinates to determine which boxes have been selected.
        """
        self.selected_boxes = (int(math.floor(self.selection[0] / (self.canvas_width / BOXES_PER_SIDE))),
                               int(math.floor(self.selection[1] / (self.canvas_height / BOXES_PER_SIDE))),
                               int(math.floor(self.selection[2] / (self.canvas_width / BOXES_PER_SIDE)) + 1),
                               int(math.floor(self.selection[3] / (self.canvas_height / BOXES_PER_SIDE)) + 1))

    def compute_new_window_size(self):
        """
        Translate the selection size to the actual screen size a window should be resized too.
        """
        self.new_window_size = (self.selected_boxes[0] * self.canvas_width / BOXES_PER_SIDE / CANVAS_SCALE,
                                self.selected_boxes[1] * self.canvas_height / BOXES_PER_SIDE / CANVAS_SCALE,
                                (self.selected_boxes[2] - self.selected_boxes[0]) * self.canvas_width / BOXES_PER_SIDE / CANVAS_SCALE,
                                (self.selected_boxes[3] - self.selected_boxes[1]) * self.canvas_height / BOXES_PER_SIDE / CANVAS_SCALE)

    def clear_buffer(self):
        """
        Clear the back-buffer.
        """
        self.buffer_pixmap.draw_rectangle(self.window.get_style().white_gc, True, 0, 0, self.canvas_width, self.canvas_height)
        self.draw_grid()

    def draw_grid(self):
        """
        Draw the selection grid onto the back-buffer.
        """
        for i in range(0, BOXES_PER_SIDE + 1):
            x = (self.canvas_width / BOXES_PER_SIDE) * i
            self.buffer_pixmap.draw_line(self.window.get_style().black_gc, x, 0, x, self.canvas_height)

        for j in range(0, BOXES_PER_SIDE + 1):
            y = (self.canvas_height / BOXES_PER_SIDE) * j
            self.buffer_pixmap.draw_line(self.window.get_style().black_gc, 0, y, self.canvas_width, y)

    def draw_selection(self):
        """
        Draw the current selection box onto the back-buffer.
        """
        style = self.window.get_style()

        self.buffer_pixmap.draw_rectangle(
            style.fg_gc[gtk.STATE_NORMAL],
            False,
            self.selection[0],
            self.selection[1],
            self.selection[2] - self.selection[0],
            self.selection[3] - self.selection[1])

    def draw_selected_boxes(self):
        """
        Draw the selected boxes in a different color.
        """
        x1 = self.selected_boxes[0] * self.canvas_width / BOXES_PER_SIDE
        y1 = self.selected_boxes[1] * self.canvas_height / BOXES_PER_SIDE
        x2 = self.selected_boxes[2] * self.canvas_width / BOXES_PER_SIDE
        y2 = self.selected_boxes[3] * self.canvas_height / BOXES_PER_SIDE

        color = self.window.get_colormap().alloc_color("#999999", False, True)

        gc = self.window.window.new_gc()
        gc.foreground = color

        self.buffer_pixmap.draw_rectangle(gc, True, int(x1), int(y1), int(x2 - x1), int(y2 - y1))

    def blit_buffer(self):
        """
        Copy the back-buffer to the visible canvas.
        """
        self.canvas.window.draw_drawable(
            self.window.get_style().fg_gc[gtk.STATE_NORMAL],
            self.buffer_pixmap,
            0, 0, 0, 0, self.canvas_width, self.canvas_height)

    def hotkey(self):
        """
        Toggle the visibility of the window.
        """
        if self.window.get_visible():
            self.window.hide()
        else:
            self.window.show()

    def bind_hotkey(self):
        """
        Bind the toggle hotkey.
        """
        keybinder.bind(HOTKEY, self.hotkey)

    def unbind_hotkey(self):
        """
        Unbind the toggle hotkey.
        """
        keybinder.unbind(HOTKEY)

if __name__ == '__main__':
    app = RationApp()
    atexit.register(app.unbind_hotkey)
    gtk.main()

<|endoftext|>