<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
import csv
import datetime
import logging
import os
import sys

import configargparse
from Bio import SeqIO

sys.path.append(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))

import camsa

if __name__ == "__main__":
    full_description = camsa.full_description_template.format(
        names=camsa.CAMSA_AUTHORS,
        affiliations=camsa.AFFILIATIONS,
        dummy=" ",
        tool="Converting FASTA formatted scaffolding results for further CAMSA processing.",
        information="For more information refer to {docs}".format(docs=camsa.CAMSA_DOCS_URL),
        contact=camsa.CONTACT)
    full_description = "=" * 80 + "\n" + full_description + "=" * 80 + "\n"
    parser = configargparse.ArgParser(description=full_description, formatter_class=configargparse.RawTextHelpFormatter)
    parser.add_argument("--version", action="version", version=camsa.VERSION)
    parser.add_argument("-o", "--output-file", metavar="OUTPUT", dest="output", type=configargparse.FileType("wt"), default=sys.stdout,
                        help="A file to which the CAMSA readable fragments lengths would be written. Standard extension is \".camsa.lengths\".\nDEFAULT: stdout")
    parser.add_argument("contigs", nargs="+", metavar="CONTIGS", type=configargparse.FileType("rt"), default=sys.stdin,
                        help="A list of input *.fasta files with contigs.\nDEFAULT: stdin")
    parser.add_argument("--c-logging-level", dest="logging_level", default=logging.INFO, type=int,
                        choices=[logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL],
                        help="Logging level for the converter.\nDEFAULT: {info}".format(info=logging.INFO))
    parser.add_argument("--c-logging-formatter-entry",
                        help="Format string for python logger.")
    args = parser.parse_args()
    start_time = datetime.datetime.now()

    logger = logging.getLogger("CAMSA.utils.fasta2camsa_lengths")
    ch = logging.StreamHandler()
    ch.setLevel(args.logging_level)
    logger.setLevel(args.logging_level)
    logger.addHandler(ch)
    logger.info(full_description)
    logger.info(parser.format_values())
    ch.setFormatter(logging.Formatter(args.c_logging_formatter_entry))
    logger.info("Starting the converting process")

    entries = {}
    for f in args.contigs:
        logger.info("Processing file: \"{file_name}\"".format(file_name=f))
        cnt = 0
        for record in SeqIO.parse(f, "fasta"):
            entries[record.id] = len(record.seq)
            cnt += 1
        logger.info("Processed {cnt} fasta records".format(cnt=cnt))
    writer = csv.writer(args.output)
    logger.info("Writing the CAMSA lengths output")
    writer.writerow(["seq_id", "seq_length"])
    for key in sorted(entries.keys()):
        writer.writerow([key, entries[key]])
    logger.info("All done!")
    end_time = datetime.datetime.now()
    logger.info("Elapsed time: {el_time}".format(el_time=str(end_time - start_time)))
<commit_msg>adding delimiter for the separation in the output as a command line argument<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
import csv
import datetime
import logging
import os
import sys

import configargparse
from Bio import SeqIO

sys.path.append(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))

import camsa

if __name__ == "__main__":
    full_description = camsa.full_description_template.format(
        names=camsa.CAMSA_AUTHORS,
        affiliations=camsa.AFFILIATIONS,
        dummy=" ",
        tool="Converting FASTA formatted scaffolding results for further CAMSA processing.",
        information="For more information refer to {docs}".format(docs=camsa.CAMSA_DOCS_URL),
        contact=camsa.CONTACT)
    full_description = "=" * 80 + "\n" + full_description + "=" * 80 + "\n"
    parser = configargparse.ArgParser(description=full_description, formatter_class=configargparse.RawTextHelpFormatter)
    parser.add_argument("--version", action="version", version=camsa.VERSION)
    parser.add_argument("-o", "--output-file", metavar="OUTPUT", dest="output", type=configargparse.FileType("wt"), default=sys.stdout,
                        help="A file to which the CAMSA readable fragments lengths would be written. Standard extension is \".camsa.lengths\".\nDEFAULT: stdout")
    parser.add_argument("contigs", nargs="+", metavar="CONTIGS", type=configargparse.FileType("rt"), default=sys.stdin,
                        help="A list of input *.fasta files with contigs.\nDEFAULT: stdin")
    parser.add_argument("--c-logging-level", dest="logging_level", default=logging.INFO, type=int,
                        choices=[logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL],
                        help="Logging level for the converter.\nDEFAULT: {info}".format(info=logging.INFO))
    parser.add_argument("--c-logging-formatter-entry",
                        help="Format string for python logger.")
    parser.add_argument("--o-delimiter", type=str, default="\t",
                        help="A single character string, used as a delimiter in the output (t)/(c)sv file.\nDEFAULT: \\t")
    args = parser.parse_args()
    start_time = datetime.datetime.now()

    logger = logging.getLogger("CAMSA.utils.fasta2camsa_lengths")
    ch = logging.StreamHandler()
    ch.setLevel(args.logging_level)
    logger.setLevel(args.logging_level)
    logger.addHandler(ch)
    logger.info(full_description)
    logger.info(parser.format_values())
    ch.setFormatter(logging.Formatter(args.c_logging_formatter_entry))
    logger.info("Starting the converting process")

    entries = {}
    for f in args.contigs:
        logger.info("Processing file: \"{file_name}\"".format(file_name=f))
        cnt = 0
        for record in SeqIO.parse(f, "fasta"):
            entries[record.id] = len(record.seq)
            cnt += 1
        logger.info("Processed {cnt} fasta records".format(cnt=cnt))
    writer = csv.writer(args.output, delimiter=args.o_delimiter)
    logger.info("Writing the CAMSA lengths output")
    writer.writerow(["seq_id", "seq_length"])
    for key in sorted(entries.keys()):
        writer.writerow([key, entries[key]])
    logger.info("All done!")
    end_time = datetime.datetime.now()
    logger.info("Elapsed time: {el_time}".format(el_time=str(end_time - start_time)))
<|endoftext|>


<commit_before>#
# Copyright 2012 New Dream Network, LLC (DreamHost)
# Copyright 2014 Hewlett-Packard Company
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Access Control Lists (ACL's) control access the API server."""

import pecan

def _has_rule(name):
    return name in pecan.request.enforcer.rules.keys()

def enforce(policy_name, request):
    """Return the user and project the request should be limited to.

    :param request: HTTP request
    :param policy_name: the policy name to validate authz against.

    """

    rule_method = "telemetry:" + policy_name
    headers = request.headers

    policy_dict = dict()
    policy_dict['roles'] = headers.get('X-Roles', "").split(",")
    policy_dict['user_id'] = (headers.get('X-User-Id'))
    policy_dict['project_id'] = (headers.get('X-Project-Id'))

    # maintain backward compat with Juno and previous by allowing the action if
    # there is no rule defined for it
    if ((_has_rule('default') or _has_rule(rule_method)) and
            not pecan.request.enforcer.enforce(rule_method, {}, policy_dict)):
        pecan.core.abort(status_code=403, detail='RBAC Authorization Failed')

# TODO(fabiog): these methods are still used because the scoping part is really
# convoluted and difficult to separate out.

def get_limited_to(headers):
    """Return the user and project the request should be limited to.

    :param headers: HTTP headers dictionary
    :return: A tuple of (user, project), set to None if there's no limit on
    one of these.

    """

    policy_dict = dict()
    policy_dict['roles'] = headers.get('X-Roles', "").split(",")
    policy_dict['user_id'] = (headers.get('X-User-Id'))
    policy_dict['project_id'] = (headers.get('X-Project-Id'))

    # maintain backward compat with Juno and previous by using context_is_admin
    # rule if the segregation rule (added in Kilo) is not defined
    rule_name = 'segregation' if _has_rule(
        'segregation') else 'context_is_admin'
    if not pecan.request.enforcer.enforce(rule_name,
                                          {},
                                          policy_dict):
        return headers.get('X-User-Id'), headers.get('X-Project-Id')

    return None, None

def get_limited_to_project(headers):
    """Return the project the request should be limited to.

    :param headers: HTTP headers dictionary
    :return: A project, or None if there's no limit on it.

    """
    return get_limited_to(headers)[1]
<commit_msg>Correct wrong description of method enforce<commit_after>#
# Copyright 2012 New Dream Network, LLC (DreamHost)
# Copyright 2014 Hewlett-Packard Company
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Access Control Lists (ACL's) control access the API server."""

import pecan

def _has_rule(name):
    return name in pecan.request.enforcer.rules.keys()

def enforce(policy_name, request):
    """Checks authorization of a rule against the request.

    :param request: HTTP request
    :param policy_name: the policy name to validate authz against.

    """

    rule_method = "telemetry:" + policy_name
    headers = request.headers

    policy_dict = dict()
    policy_dict['roles'] = headers.get('X-Roles', "").split(",")
    policy_dict['user_id'] = (headers.get('X-User-Id'))
    policy_dict['project_id'] = (headers.get('X-Project-Id'))

    # maintain backward compat with Juno and previous by allowing the action if
    # there is no rule defined for it
    if ((_has_rule('default') or _has_rule(rule_method)) and
            not pecan.request.enforcer.enforce(rule_method, {}, policy_dict)):
        pecan.core.abort(status_code=403, detail='RBAC Authorization Failed')

# TODO(fabiog): these methods are still used because the scoping part is really
# convoluted and difficult to separate out.

def get_limited_to(headers):
    """Return the user and project the request should be limited to.

    :param headers: HTTP headers dictionary
    :return: A tuple of (user, project), set to None if there's no limit on
    one of these.

    """

    policy_dict = dict()
    policy_dict['roles'] = headers.get('X-Roles', "").split(",")
    policy_dict['user_id'] = (headers.get('X-User-Id'))
    policy_dict['project_id'] = (headers.get('X-Project-Id'))

    # maintain backward compat with Juno and previous by using context_is_admin
    # rule if the segregation rule (added in Kilo) is not defined
    rule_name = 'segregation' if _has_rule(
        'segregation') else 'context_is_admin'
    if not pecan.request.enforcer.enforce(rule_name,
                                          {},
                                          policy_dict):
        return headers.get('X-User-Id'), headers.get('X-Project-Id')

    return None, None

def get_limited_to_project(headers):
    """Return the project the request should be limited to.

    :param headers: HTTP headers dictionary
    :return: A project, or None if there's no limit on it.

    """
    return get_limited_to(headers)[1]
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2017-2018 ARM Limited
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import sys
import shutil
from argparse import ArgumentParser

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                    os.pardir, os.pardir))
sys.path.insert(0, ROOT)
from tools.targets import Target, TARGET_MAP, TARGET_NAMES

MAKE_PY_LOCATTION = os.path.join(ROOT, 'tools', 'make.py')
TEST_PY_LOCATTION = os.path.join(ROOT, 'tools', 'test.py')
TFM_MBED_APP = os.path.join(ROOT, 'tools', 'psa', 'tfm', 'mbed_app.json')
MBED_PSA_TESTS = '*psa-spm*,*psa-crypto_access_control'
TFM_TESTS = {
    '*psa-spm_smoke': ['USE_PSA_TEST_PARTITIONS', 'USE_SMOKE_TESTS_PART1'],
    '*psa-spm_client': ['USE_PSA_TEST_PARTITIONS', 'USE_CLIENT_TESTS_PART1'],
    '*psa-spm_server': ['USE_PSA_TEST_PARTITIONS', 'USE_SERVER_TESTS_PART1',
                        'USE_SERVER_TESTS_PART2'],
    '*psa-crypto_access_control': ['USE_PSA_TEST_PARTITIONS',
                                   'USE_CRYPTO_ACL_TEST']
}

def _psa_backend(target):
    """
    Returns a target PSA backend.

    :param target: Target name as in targets.json
    :return:  PSA backend as string (TFM/MBED_SPM)
    """
    return 'TFM' if 'TFM' in Target.get_target(target).labels else 'MBED_SPM'

def _get_target_info(target):
    """
    Creates a PSA target tuple with default toolchain and
    artifact delivery directory.

    :param target: Target name.
    :return: tuple (target, toolchain, delivery directory).
    """
    delivery_dir = os.path.join(ROOT, 'targets',
                                TARGET_MAP[target].delivery_dir)

    if not os.path.exists(delivery_dir):
        raise Exception("{} does not have delivery_dir".format(target))

    return tuple([TARGET_MAP[target].name,
                  TARGET_MAP[target].default_toolchain,
                  delivery_dir])

def get_mbed_official_psa_release(target=None):
    """
    Creates a list of PSA targets with default toolchain and
    artifact delivery directory.

    :param target: Ask for specific target, None for all targets.
    :return: List of tuples (target, toolchain, delivery directory).
    """
    psa_targets_release_list = []
    psa_secure_targets = [t for t in TARGET_NAMES if
                          Target.get_target(t).is_PSA_secure_target]
    if target is not None:
        if target not in psa_secure_targets:
            raise Exception("{} is not a PSA secure target".format(target))
        psa_targets_release_list.append(_get_target_info(target))
    else:
        for t in psa_secure_targets:
            psa_targets_release_list.append(_get_target_info(target))

    return psa_targets_release_list

def create_mbed_ignore(build_dir):
    """
    Creates a .mbedignore file in a given directory.

    :param build_dir: Directory to create .mbedignore file.
    """
    with open(os.path.join(build_dir, '.mbedignore'), 'w') as f:
        f.write('*\n')

def build_mbed_spm_platform(target, toolchain, profile='release'):
    """
    Builds Secure images for MBED-SPM target.

    :param target: target to be built.
    :param toolchain: toolchain to be used.
    :param profile: build profile.
    """
    subprocess.check_call([
        sys.executable, '-u', TEST_PY_LOCATTION,
        '--greentea',
        '--profile', profile,
        '-t', toolchain,
        '-m', target,
        '--source', ROOT,
        '--build', os.path.join(ROOT, 'BUILD', 'tests', target),
        '--test-spec', os.path.join(ROOT, 'BUILD', 'tests',
                                    target, 'test_spec.json'),
        '--build-data', os.path.join(ROOT, 'BUILD', 'tests',
                                     target, 'build_data.json'),
        '-n', MBED_PSA_TESTS
    ], stdout=subprocess.PIPE)

    subprocess.check_call([
        sys.executable, '-u', MAKE_PY_LOCATTION,
        '-t', toolchain,
        '-m', target,
        '--profile', profile,
        '--source', ROOT,
        '--build', os.path.join(ROOT, 'BUILD', target),
        '--artifact-name', 'psa_release_1.0'
    ], stdout=subprocess.PIPE)

def _tfm_test_defines(test):
    """
    Creates a define list to enable test partitions on TF-M.

    :param test: Test name.
    :return: List of defines with a leading -D.
    """
    return ['-D{}'.format(define) for define in TFM_TESTS[test]]

def build_tfm_platform(target, toolchain, profile='release'):
    """
    Builds Secure images for TF-M target.

    :param target: target to be built.
    :param toolchain: toolchain to be used.
    :param profile: build profile.
    """
    for test in TFM_TESTS.keys():
        subprocess.check_call([
            sys.executable, '-u', TEST_PY_LOCATTION,
            '--greentea',
            '--profile', profile,
            '-t', toolchain,
            '-m', target,
            '--source', ROOT,
            '--build', os.path.join(ROOT, 'BUILD', 'tests', target),
            '--test-spec', os.path.join(ROOT, 'BUILD', 'tests',
                                        target, 'test_spec.json'),
            '--build-data', os.path.join(ROOT, 'BUILD', 'tests',
                                         target, 'build_data.json'),
            '--app-config', TFM_MBED_APP, '-n', test] + _tfm_test_defines(test),
                              stdout=subprocess.PIPE)

    subprocess.check_call([
        sys.executable, '-u', MAKE_PY_LOCATTION,
        '-t', toolchain,
        '-m', target,
        '--profile', profile,
        '--source', ROOT,
        '--build', os.path.join(ROOT, 'BUILD', target),
        '--app-config', TFM_MBED_APP
    ], stdout=subprocess.PIPE)

def commit_binaries(target, delivery_dir):
    """
    Commits changes in secure binaries.

    :param target: Target name.
    :param delivery_dir: Secure images should be moved to this folder
    by the build system.
    """
    changes_made = subprocess.call([
        'git',
        '-C', ROOT,
        'diff', '--exit-code',
        delivery_dir
    ], stdout=subprocess.PIPE)

    if changes_made:
        subprocess.check_call([
            'git',
            '-C', ROOT,
            'add', os.path.relpath(delivery_dir, ROOT)
        ], stdout=subprocess.PIPE)

        commit_message = '-m\"Update secure binaries for {}\"'.format(target)
        subprocess.check_call([
            'git',
            '-C', ROOT,
            'commit',
            commit_message
        ], stdout=subprocess.PIPE)

def build_psa_platform(target, toolchain, delivery_dir, debug=False,
                       git_commit=False):
    """
    Calls the correct build function and commits if requested.

    :param target: Target name.
    :param toolchain: Toolchain to be used.
    :param delivery_dir: Artifact directory, where images should be placed.
    :param debug: Build with debug profile.
    :param git_commit: Commit the changes.
    """
    profile = 'debug' if debug else 'release'
    if _psa_backend(target) is 'TFM':
        build_tfm_platform(target, toolchain, profile)
    else:
        build_mbed_spm_platform(target, toolchain, profile)

    if git_commit:
        commit_binaries(target, delivery_dir)

def get_parser():
    parser = ArgumentParser()
    parser.add_argument("-m", "--mcu",
                        help="build for the given MCU",
                        default=None,
                        metavar="MCU")

    parser.add_argument("-d", "--debug",
                        help="set build profile to debug",
                        action="store_true",
                        default=False)

    parser.add_argument("--commit",
                        help="create a git commit for each platform",
                        action="store_true",
                        default=False)

    return parser

def prep_build_dir():
    """
    Creates a clean BUILD directory
    """
    build_dir = os.path.join(ROOT, 'BUILD')
    if os.path.exists(build_dir):
        shutil.rmtree(build_dir)

    os.makedirs(build_dir)
    create_mbed_ignore(build_dir)

def main():
    parser = get_parser()
    options = parser.parse_args()
    prep_build_dir()
    psa_platforms_list = get_mbed_official_psa_release(options.mcu)

    for target, tc, directory in psa_platforms_list:
        build_psa_platform(target, tc, directory, options.debug, options.commit)

if __name__ == '__main__':
    main()
<commit_msg>Minor changes fixing the script from getting stuck<commit_after>#!/usr/bin/python
# Copyright (c) 2017-2018 ARM Limited
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import sys
import shutil
from argparse import ArgumentParser

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                    os.pardir, os.pardir))
sys.path.insert(0, ROOT)
from tools.targets import Target, TARGET_MAP, TARGET_NAMES

MAKE_PY_LOCATTION = os.path.join(ROOT, 'tools', 'make.py')
TEST_PY_LOCATTION = os.path.join(ROOT, 'tools', 'test.py')
TFM_MBED_APP = os.path.join(ROOT, 'tools', 'psa', 'tfm', 'mbed_app.json')
MBED_PSA_TESTS = '*psa-spm*,*psa-crypto_access_control'
TFM_TESTS = {
    '*psa-spm_smoke': ['USE_PSA_TEST_PARTITIONS', 'USE_SMOKE_TESTS_PART1'],
    '*psa-spm_client': ['USE_PSA_TEST_PARTITIONS', 'USE_CLIENT_TESTS_PART1'],
    '*psa-spm_server': ['USE_PSA_TEST_PARTITIONS', 'USE_SERVER_TESTS_PART1',
                        'USE_SERVER_TESTS_PART2'],
    '*psa-crypto_access_control': ['USE_PSA_TEST_PARTITIONS',
                                   'USE_CRYPTO_ACL_TEST']
}

def _psa_backend(target):
    """
    Returns a target PSA backend.

    :param target: Target name as in targets.json
    :return:  PSA backend as string (TFM/MBED_SPM)
    """
    return 'TFM' if 'TFM' in Target.get_target(target).labels else 'MBED_SPM'

def _get_target_info(target):
    """
    Creates a PSA target tuple with default toolchain and
    artifact delivery directory.

    :param target: Target name.
    :return: tuple (target, toolchain, delivery directory).
    """
    delivery_dir = os.path.join(ROOT, 'targets',
                                TARGET_MAP[target].delivery_dir)

    if not os.path.exists(delivery_dir):
        raise Exception("{} does not have delivery_dir".format(target))

    return tuple([TARGET_MAP[target].name,
                  TARGET_MAP[target].default_toolchain,
                  delivery_dir])

def get_mbed_official_psa_release(target=None):
    """
    Creates a list of PSA targets with default toolchain and
    artifact delivery directory.

    :param target: Ask for specific target, None for all targets.
    :return: List of tuples (target, toolchain, delivery directory).
    """
    psa_targets_release_list = []
    psa_secure_targets = [t for t in TARGET_NAMES if
                          Target.get_target(t).is_PSA_secure_target]
    if target is not None:
        if target not in psa_secure_targets:
            raise Exception("{} is not a PSA secure target".format(target))
        psa_targets_release_list.append(_get_target_info(target))
    else:
        for t in psa_secure_targets:
            psa_targets_release_list.append(_get_target_info(target))

    return psa_targets_release_list

def create_mbed_ignore(build_dir):
    """
    Creates a .mbedignore file in a given directory.

    :param build_dir: Directory to create .mbedignore file.
    """
    with open(os.path.join(build_dir, '.mbedignore'), 'w') as f:
        f.write('*\n')

def build_mbed_spm_platform(target, toolchain, profile='release'):
    """
    Builds Secure images for MBED-SPM target.

    :param target: target to be built.
    :param toolchain: toolchain to be used.
    :param profile: build profile.
    """
    subprocess.check_call([
        sys.executable, TEST_PY_LOCATTION,
        '--greentea',
        '--profile', profile,
        '-t', toolchain,
        '-m', target,
        '--source', ROOT,
        '--build', os.path.join(ROOT, 'BUILD', 'tests', target),
        '--test-spec', os.path.join(ROOT, 'BUILD', 'tests',
                                    target, 'test_spec.json'),
        '--build-data', os.path.join(ROOT, 'BUILD', 'tests',
                                     target, 'build_data.json'),
        '-n', MBED_PSA_TESTS
    ])

    subprocess.check_call([
        sys.executable, MAKE_PY_LOCATTION,
        '-t', toolchain,
        '-m', target,
        '--profile', profile,
        '--source', ROOT,
        '--build', os.path.join(ROOT, 'BUILD', target),
        '--artifact-name', 'psa_release_1.0'
    ])

def _tfm_test_defines(test):
    """
    Creates a define list to enable test partitions on TF-M.

    :param test: Test name.
    :return: List of defines with a leading -D.
    """
    return ['-D{}'.format(define) for define in TFM_TESTS[test]]

def build_tfm_platform(target, toolchain, profile='release'):
    """
    Builds Secure images for TF-M target.

    :param target: target to be built.
    :param toolchain: toolchain to be used.
    :param profile: build profile.
    """
    for test in TFM_TESTS.keys():
        subprocess.check_call([
            sys.executable, TEST_PY_LOCATTION,
            '--greentea',
            '--profile', profile,
            '-t', toolchain,
            '-m', target,
            '--source', ROOT,
            '--build', os.path.join(ROOT, 'BUILD', 'tests', target),
            '--test-spec', os.path.join(ROOT, 'BUILD', 'tests',
                                        target, 'test_spec.json'),
            '--build-data', os.path.join(ROOT, 'BUILD', 'tests',
                                         target, 'build_data.json'),
            '--app-config', TFM_MBED_APP, '-n', test] + _tfm_test_defines(test),
                              stdout=subprocess.PIPE)

    subprocess.check_call([
        sys.executable, MAKE_PY_LOCATTION,
        '-t', toolchain,
        '-m', target,
        '--profile', profile,
        '--source', ROOT,
        '--build', os.path.join(ROOT, 'BUILD', target),
        '--app-config', TFM_MBED_APP
    ])

def commit_binaries(target, delivery_dir):
    """
    Commits changes in secure binaries.

    :param target: Target name.
    :param delivery_dir: Secure images should be moved to this folder
    by the build system.
    """
    changes_made = subprocess.call([
        'git',
        '-C', ROOT,
        'diff', '--exit-code', '--quiet',
        delivery_dir
    ])

    if changes_made:
        subprocess.check_call([
            'git',
            '-C', ROOT,
            'add', os.path.relpath(delivery_dir, ROOT)
        ])

        commit_message = '-m\"Update secure binaries for {}\"'.format(target)
        subprocess.check_call([
            'git',
            '-C', ROOT,
            'commit',
            commit_message
        ])

def build_psa_platform(target, toolchain, delivery_dir, debug=False,
                       git_commit=False):
    """
    Calls the correct build function and commits if requested.

    :param target: Target name.
    :param toolchain: Toolchain to be used.
    :param delivery_dir: Artifact directory, where images should be placed.
    :param debug: Build with debug profile.
    :param git_commit: Commit the changes.
    """
    profile = 'debug' if debug else 'release'
    if _psa_backend(target) is 'TFM':
        build_tfm_platform(target, toolchain, profile)
    else:
        build_mbed_spm_platform(target, toolchain, profile)

    if git_commit:
        commit_binaries(target, delivery_dir)

def get_parser():
    parser = ArgumentParser()
    parser.add_argument("-m", "--mcu",
                        help="build for the given MCU",
                        default=None,
                        metavar="MCU")

    parser.add_argument("-d", "--debug",
                        help="set build profile to debug",
                        action="store_true",
                        default=False)

    parser.add_argument("--commit",
                        help="create a git commit for each platform",
                        action="store_true",
                        default=False)

    return parser

def prep_build_dir():
    """
    Creates a clean BUILD directory
    """
    build_dir = os.path.join(ROOT, 'BUILD')
    if os.path.exists(build_dir):
        shutil.rmtree(build_dir)

    os.makedirs(build_dir)
    create_mbed_ignore(build_dir)

def main():
    parser = get_parser()
    options = parser.parse_args()
    prep_build_dir()
    psa_platforms_list = get_mbed_official_psa_release(options.mcu)

    for target, tc, directory in psa_platforms_list:
        build_psa_platform(target, tc, directory, options.debug, options.commit)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from flask import Blueprint, render_template, current_app, abort, make_response

import cea.inputlocator
import cea.plots
import cea.plots.categories

import re
import os

blueprint = Blueprint(
    'plots_blueprint',
    __name__,
    url_prefix='/plots',
    template_folder='templates',
    static_folder='static',
)

def script_suggestions(locator_names):
    """Return a list of CeaScript objects that produce the output for each locator name"""
    import cea.scripts
    schemas = cea.scripts.schemas()
    script_names = []
    for name in locator_names:
        script_names.extend(schemas[name]['created_by'])
    return [cea.scripts.by_name(n) for n in sorted(set(script_names))]

def load_plot(dashboard, plot_index):
    """Load a plot from the dashboard_yml"""
    cea_config = current_app.cea_config
    dashboards = cea.plots.read_dashboards(cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard]
    plot = dashboard.plots[plot_index]
    return plot

@blueprint.route('/div/<int:dashboard_index>/<int:plot_index>')
def route_div(dashboard_index, plot_index):
    """Return the plot as a div to be used in an AJAX call"""
    try:
        plot = load_plot(dashboard_index, plot_index)
    except Exception as ex:
        return abort(500, ex)
    if not plot.missing_input_files():
        plot_div = plot.plot_div()
        # BUGFIX for (#2102 - Can't add the same plot twice in a dashboard)
        # update id of div to include dashboard_index and plot_index
        if plot_div.startswith("<div id="):
            div_id = re.match('<div id="([0-9a-f-]+)"', plot_div).group(1)
            plot_div = plot_div.replace(div_id, "{div_id}-{dashboard_index}-{plot_index}".format(
                div_id=div_id, dashboard_index=dashboard_index, plot_index=plot_index))
        return make_response(plot_div, 200)
    else:
        return render_template('missing_input_files.html',
                               missing_input_files=[lm(*args) for lm, args in plot.missing_input_files()],
                               script_suggestions=script_suggestions(lm.__name__ for lm, _ in plot.missing_input_files())), 404

@blueprint.route('/plot/<int:dashboard_index>/<int:plot_index>')
def route_plot(dashboard_index, plot_index):
    try:
        plot = load_plot(dashboard_index, plot_index)
        plot_title = plot.title
        if 'scenario-name' in plot.parameters:
            plot_title += ' - {}'.format(plot.parameters['scenario-name'])
    except Exception as ex:
        return abort(500, ex)

    if not plot.missing_input_files():
        plot_div = plot.plot_div()
    else:
        return render_template('missing_input_files.html',
                               missing_input_files=[lm(*args) for lm, args in plot.missing_input_files()],
                               script_suggestions=script_suggestions(
                                   lm.__name__ for lm, _ in plot.missing_input_files())), 404

    return render_template('plot.html', plot_div=plot_div, plot_title=plot_title)

@blueprint.app_errorhandler(500)
def internal_error(error):
    import traceback
    error_trace = traceback.format_exc()
    return error_trace, 500
<commit_msg>Catch MissingInputDataException instead of checking for missing files<commit_after>from flask import Blueprint, render_template, current_app, abort, make_response

import cea.inputlocator
import cea.plots
import cea.plots.categories

import re
import os

from cea import MissingInputDataException

blueprint = Blueprint(
    'plots_blueprint',
    __name__,
    url_prefix='/plots',
    template_folder='templates',
    static_folder='static',
)

def script_suggestions(locator_names):
    """Return a list of CeaScript objects that produce the output for each locator name"""
    import cea.scripts
    schemas = cea.scripts.schemas()
    script_names = []
    for name in locator_names:
        script_names.extend(schemas[name]['created_by'])
    return [cea.scripts.by_name(n) for n in sorted(set(script_names))]

def load_plot(dashboard, plot_index):
    """Load a plot from the dashboard_yml"""
    cea_config = current_app.cea_config
    dashboards = cea.plots.read_dashboards(cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard]
    plot = dashboard.plots[plot_index]
    return plot

def render_missing_data(missing_files):
    return render_template('missing_input_files.html',
                           missing_input_files=[lm(*args) for lm, args in missing_files],
                           script_suggestions=script_suggestions(lm.__name__ for lm, _ in missing_files)), 404

@blueprint.route('/div/<int:dashboard_index>/<int:plot_index>')
def route_div(dashboard_index, plot_index):
    """Return the plot as a div to be used in an AJAX call"""
    plot = load_plot(dashboard_index, plot_index)
    try:
        plot_div = plot.plot_div()
    except MissingInputDataException:
        return render_missing_data(plot.missing_input_files())
    # BUGFIX for (#2102 - Can't add the same plot twice in a dashboard)
    # update id of div to include dashboard_index and plot_index
    if plot_div.startswith("<div id="):
        div_id = re.match('<div id="([0-9a-f-]+)"', plot_div).group(1)
        plot_div = plot_div.replace(div_id, "{div_id}-{dashboard_index}-{plot_index}".format(
            div_id=div_id, dashboard_index=dashboard_index, plot_index=plot_index))
    return make_response(plot_div, 200)

@blueprint.route('/plot/<int:dashboard_index>/<int:plot_index>')
def route_plot(dashboard_index, plot_index):
    plot = load_plot(dashboard_index, plot_index)
    plot_title = plot.title
    if 'scenario-name' in plot.parameters:
        plot_title += ' - {}'.format(plot.parameters['scenario-name'])
    try:
        plot_div = plot.plot_div()
    except MissingInputDataException:
        return render_missing_data(plot.missing_input_files())
    return render_template('plot.html', plot_div=plot_div, plot_title=plot_title)

@blueprint.app_errorhandler(500)
def internal_error(error):
    import traceback
    error_trace = traceback.format_exc()
    return error_trace, 500
<|endoftext|>


<commit_before>

def normalize_fitnesses(scaler_dict, fitnesses_population):
    number_of_objectives = scaler_dict['NOBJ']
    fitness_population_scaled = []
    for ind in fitnesses_population:
        fitness_individual_scaled = []
        for objective in range(number_of_objectives):
            min_value = scaler_dict['min'][objective]
            max_value = scaler_dict['max'][objective]
            fitness_individual_scaled.append(minmax_scaler(ind[objective], min_value, max_value))
        fitness_population_scaled.append(tuple(fitness_individual_scaled))

    return fitness_population_scaled

def minmax_scaler(value, min_value, max_value):
    return (value - min_value) / (max_value - min_value)

def scaler_for_normalization(number_of_objectives, fitnesses):
    max_ob = []
    min_ob = []
    for objective in range(number_of_objectives):
        list_fitness_objective = [x[objective] for x in fitnesses]
        max_ob.append(max(list_fitness_objective))
        min_ob.append(min(list_fitness_objective, 0.0))  # in case there are negative, select the negative

    # Output to scale data inside the fitness funcion from the second generation on
    scaler_dict = {'max': max_ob, 'min': min_ob, 'NOBJ': number_of_objectives}

    return scaler_dict
<commit_msg>fixed min bug according to talk with jimeno<commit_after>

def normalize_fitnesses(scaler_dict, fitnesses_population):
    number_of_objectives = scaler_dict['NOBJ']
    fitness_population_scaled = []
    for ind in fitnesses_population:
        fitness_individual_scaled = []
        for objective in range(number_of_objectives):
            min_value = scaler_dict['min'][objective]
            max_value = scaler_dict['max'][objective]
            fitness_individual_scaled.append(minmax_scaler(ind[objective], min_value, max_value))
        fitness_population_scaled.append(tuple(fitness_individual_scaled))

    return fitness_population_scaled

def minmax_scaler(value, min_value, max_value):
    return (value - min_value) / (max_value - min_value)

def scaler_for_normalization(number_of_objectives, fitnesses):
    max_ob = []
    min_ob = []
    for objective in range(number_of_objectives):
        list_fitness_objective = [x[objective] for x in fitnesses]
        max_ob.append(max(list_fitness_objective))
        min_ob.append(min(list_fitness_objective, default=0.0))
        min_ob.append(min(min(list_fitness_objective), 0.0))  # select lowest negative value or 0.0 if only positives

    # Output to scale data inside the fitness funcion from the second generation on
    scaler_dict = {'max': max_ob, 'min': min_ob, 'NOBJ': number_of_objectives}

    return scaler_dict
<|endoftext|>


<commit_before>import argparse
import logging
from distutils.util import strtobool

from . import exc
from . import lsb, hosts
from .cliutil import priority
from .sudo_pushy import get_transport
from .util.decorators import remote_compile

LOG = logging.getLogger(__name__)

def check_ceph_installed():
    """
    Check if the ceph packages are installed by looking for the
    presence of the ceph command.
    """
    import subprocess

    args = [ 'which', 'ceph', ]
    process = subprocess.Popen(
        args=args,
        )
    lsb_release_path, _ = process.communicate()
    return process.wait()

def uninstall_suse(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'libcephfs1',
        'librados2',
        'librbd1',
        ]
    args = [
        'zypper',
        '--non-interactive',
        '--quiet',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_debian(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'apt-get',
        '-q',
        'remove',
        '-f',
        '-y',
        '--force-yes',
        ]
    if arg_purge:
        args.append('--purge')
    args.append('--')
    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_fedora(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        ]
    args = [
        'yum',
        '-q',
        '-y',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_centos(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        ]
    args = [
        'yum',
        '-q',
        '-y',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_debian(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'apt-get',
        '-q',
        'remove',
        '-f',
        '-y',
        '--force-yes',
        ]
    if arg_purge:
        args.append('--purge')
    args.append('--')
    args.extend(packages)
    subprocess.check_call(args=args)

def purge_data_any():
    import subprocess
    import os.path

    subprocess.call(args=[
            'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
            ])
    if os.path.exists('/var/lib/ceph'):
        subprocess.check_call(args=[
                'find', '/var/lib/ceph',
                '-mindepth', '1',
                '-maxdepth', '2',
                '-type', 'd',
                '-exec', 'umount', '{}', ';',
                ])
        subprocess.check_call(args=[
                'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
                ])
    subprocess.check_call(args=[
            'rm', '-rf', '--one-file-system', '--', '/etc/ceph',
            ])

def install(args):
    version = getattr(args, args.version_kind)
    version_str = args.version_kind
    if version:
        version_str += ' version {version}'.format(version=version)
    LOG.debug(
        'Installing %s on cluster %s hosts %s',
        version_str,
        args.cluster,
        ' '.join(args.host),
        )
    for hostname in args.host:
        # TODO username
        LOG.debug('Detecting platform for host %s ...', hostname)
        distro = hosts.get(hostname)
        LOG.info('Distro info: %s %s %s', distro.name, distro.release, distro.codename)
        rlogger = logging.getLogger(hostname)
        rlogger.info('installing ceph on %s' % hostname)
        distro.install(distro, rlogger, args.version_kind, version, args.pkgs_only)
        distro.sudo_conn.close()

def uninstall(args):
    LOG.debug(
        'Uninstalling on cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            uninstall_r = sudo.compile(uninstall_debian)
        elif distro == 'CentOS' or distro == 'Scientific' or distro.startswith('RedHat'):
            uninstall_r = sudo.compile(uninstall_centos)
        elif distro == 'Fedora':
            uninstall_r = sudo.compile(uninstall_fedora)
        elif (distro == 'SUSE LINUX'):
            uninstall_r = sudo.compile(uninstall_suse)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        LOG.debug('Uninstalling on host %s ...', hostname)
        uninstall_r()
        sudo.close()

def purge(args):
    LOG.debug(
        'Purging from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if distro == 'Debian' or distro == 'Ubuntu':
            purge_r = sudo.compile(uninstall_debian)
        elif distro == 'CentOS' or distro == 'Scientific' or distro.startswith('RedHat'):
            purge_r = sudo.compile(uninstall_centos)
        elif distro == 'Fedora':
            purge_r = sudo.compile(uninstall_fedora)
        elif (distro == 'SUSE LINUX'):
            purge_r = sudo.compile(uninstall_suse)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        LOG.debug('Purging host %s ...', hostname)
        purge_r(arg_purge=True)
        sudo.close()

def purge_data(args):
    LOG.debug(
        'Purging data from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    installed_hosts=[]
    for hostname in args.host:
        sudo = args.pushy(get_transport(hostname))
        check_ceph_installed_r  = sudo.compile(check_ceph_installed)
        status = check_ceph_installed_r()
        if status == 0:
            installed_hosts.append(hostname)
        sudo.close()

    if installed_hosts:
        print "ceph is still installed on: ", installed_hosts
        answer=raw_input("Continue (y/n)")
        if not strtobool(answer):
            return

    for hostname in args.host:
        # TODO username
        sudo = args.pushy(get_transport(hostname))

        LOG.debug('Purging data from host %s ...', hostname)
        purge_data_any_r = sudo.compile(purge_data_any)
        purge_data_any_r()
        sudo.close()

class StoreVersion(argparse.Action):
    """
    Like ``"store"`` but also remember which one of the exclusive
    options was set.

    There are three kinds of versions: stable, testing and dev.
    This sets ``version_kind`` to be the right one of the above.

    This kludge essentially lets us differentiate explicitly set
    values from defaults.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)
        namespace.version_kind = self.dest

@priority(20)
def make(parser):
    """
    Install Ceph packages on remote hosts.
    """

    version = parser.add_mutually_exclusive_group()

    version.add_argument(
        '--stable',
        nargs='?',
        action=StoreVersion,
        choices=[
            'bobtail',
            'cuttlefish',
            'dumpling',
            'emperor',
            ],
        metavar='CODENAME',
        help='install a release known as CODENAME (done by default) (default: %(default)s)',
        )

    version.add_argument(
        '--testing',
        nargs=0,
        action=StoreVersion,
        help='install the latest development release',
        )

    version.add_argument(
        '--dev',
        nargs='?',
        action=StoreVersion,
        const='master',
        metavar='BRANCH_OR_TAG',
        help='install a bleeding edge build from Git branch or tag (default: %(default)s)',
        )

    version.add_argument(
        '--pkgs-only',
        action='store_true',
        default=False,
        help='install packages only, skipping repo setup',
        )

    version.set_defaults(
        func=install,
        stable='dumpling',
        dev='master',
        version_kind='stable',
        )

    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to install on',
        )
    parser.set_defaults(
        func=install,
        )

@priority(80)
def make_uninstall(parser):
    """
    Remove Ceph packages from remote hosts.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to uninstall Ceph from',
        )
    parser.set_defaults(
        func=uninstall,
        )

@priority(80)
def make_purge(parser):
    """
    Remove Ceph packages from remote hosts and purge all data.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph from',
        )
    parser.set_defaults(
        func=purge,
        )

@priority(80)
def make_purge_data(parser):
    """
    Purge (delete, destroy, discard, shred) any Ceph data from /var/lib/ceph
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph data from',
        )
    parser.set_defaults(
        func=purge_data,
        )
<commit_msg>add boolean flags for mangling repos<commit_after>import argparse
import logging
from distutils.util import strtobool

from . import exc
from . import lsb, hosts
from .cliutil import priority
from .sudo_pushy import get_transport
from .util.decorators import remote_compile

LOG = logging.getLogger(__name__)

def check_ceph_installed():
    """
    Check if the ceph packages are installed by looking for the
    presence of the ceph command.
    """
    import subprocess

    args = [ 'which', 'ceph', ]
    process = subprocess.Popen(
        args=args,
        )
    lsb_release_path, _ = process.communicate()
    return process.wait()

def uninstall_suse(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'libcephfs1',
        'librados2',
        'librbd1',
        ]
    args = [
        'zypper',
        '--non-interactive',
        '--quiet',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_debian(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'apt-get',
        '-q',
        'remove',
        '-f',
        '-y',
        '--force-yes',
        ]
    if arg_purge:
        args.append('--purge')
    args.append('--')
    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_fedora(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        ]
    args = [
        'yum',
        '-q',
        '-y',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_centos(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        ]
    args = [
        'yum',
        '-q',
        '-y',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_debian(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'apt-get',
        '-q',
        'remove',
        '-f',
        '-y',
        '--force-yes',
        ]
    if arg_purge:
        args.append('--purge')
    args.append('--')
    args.extend(packages)
    subprocess.check_call(args=args)

def purge_data_any():
    import subprocess
    import os.path

    subprocess.call(args=[
            'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
            ])
    if os.path.exists('/var/lib/ceph'):
        subprocess.check_call(args=[
                'find', '/var/lib/ceph',
                '-mindepth', '1',
                '-maxdepth', '2',
                '-type', 'd',
                '-exec', 'umount', '{}', ';',
                ])
        subprocess.check_call(args=[
                'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
                ])
    subprocess.check_call(args=[
            'rm', '-rf', '--one-file-system', '--', '/etc/ceph',
            ])

def install(args):
    version = getattr(args, args.version_kind)
    version_str = args.version_kind
    if version:
        version_str += ' version {version}'.format(version=version)
    LOG.debug(
        'Installing %s on cluster %s hosts %s',
        version_str,
        args.cluster,
        ' '.join(args.host),
        )
    for hostname in args.host:
        # TODO username
        LOG.debug('Detecting platform for host %s ...', hostname)
        distro = hosts.get(hostname)
        LOG.info('Distro info: %s %s %s', distro.name, distro.release, distro.codename)
        rlogger = logging.getLogger(hostname)
        rlogger.info('installing ceph on %s' % hostname)
        distro.install(distro, rlogger, args.version_kind, version, args.adjust_repos)
        distro.sudo_conn.close()

def uninstall(args):
    LOG.debug(
        'Uninstalling on cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            uninstall_r = sudo.compile(uninstall_debian)
        elif distro == 'CentOS' or distro == 'Scientific' or distro.startswith('RedHat'):
            uninstall_r = sudo.compile(uninstall_centos)
        elif distro == 'Fedora':
            uninstall_r = sudo.compile(uninstall_fedora)
        elif (distro == 'SUSE LINUX'):
            uninstall_r = sudo.compile(uninstall_suse)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        LOG.debug('Uninstalling on host %s ...', hostname)
        uninstall_r()
        sudo.close()

def purge(args):
    LOG.debug(
        'Purging from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if distro == 'Debian' or distro == 'Ubuntu':
            purge_r = sudo.compile(uninstall_debian)
        elif distro == 'CentOS' or distro == 'Scientific' or distro.startswith('RedHat'):
            purge_r = sudo.compile(uninstall_centos)
        elif distro == 'Fedora':
            purge_r = sudo.compile(uninstall_fedora)
        elif (distro == 'SUSE LINUX'):
            purge_r = sudo.compile(uninstall_suse)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        LOG.debug('Purging host %s ...', hostname)
        purge_r(arg_purge=True)
        sudo.close()

def purge_data(args):
    LOG.debug(
        'Purging data from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    installed_hosts=[]
    for hostname in args.host:
        sudo = args.pushy(get_transport(hostname))
        check_ceph_installed_r  = sudo.compile(check_ceph_installed)
        status = check_ceph_installed_r()
        if status == 0:
            installed_hosts.append(hostname)
        sudo.close()

    if installed_hosts:
        print "ceph is still installed on: ", installed_hosts
        answer=raw_input("Continue (y/n)")
        if not strtobool(answer):
            return

    for hostname in args.host:
        # TODO username
        sudo = args.pushy(get_transport(hostname))

        LOG.debug('Purging data from host %s ...', hostname)
        purge_data_any_r = sudo.compile(purge_data_any)
        purge_data_any_r()
        sudo.close()

class StoreVersion(argparse.Action):
    """
    Like ``"store"`` but also remember which one of the exclusive
    options was set.

    There are three kinds of versions: stable, testing and dev.
    This sets ``version_kind`` to be the right one of the above.

    This kludge essentially lets us differentiate explicitly set
    values from defaults.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)
        namespace.version_kind = self.dest

@priority(20)
def make(parser):
    """
    Install Ceph packages on remote hosts.
    """

    version = parser.add_mutually_exclusive_group()

    version.add_argument(
        '--stable',
        nargs='?',
        action=StoreVersion,
        choices=[
            'bobtail',
            'cuttlefish',
            'dumpling',
            'emperor',
            ],
        metavar='CODENAME',
        help='install a release known as CODENAME (done by default) (default: %(default)s)',
    )

    version.add_argument(
        '--testing',
        nargs=0,
        action=StoreVersion,
        help='install the latest development release',
    )

    version.add_argument(
        '--dev',
        nargs='?',
        action=StoreVersion,
        const='master',
        metavar='BRANCH_OR_TAG',
        help='install a bleeding edge build from Git branch or tag (default: %(default)s)',
    )

    version.add_argument(
        '--adjust-repos',
        dest='adjust_repos',
        action='store_true',
        help='install packages modifying source repos',
    )

    version.add_argument(
        '--no-adjust-repos',
        dest='adjust_repos',
        action='store_false',
        help='install packages without modifying source repos',
    )

    version.set_defaults(
        func=install,
        stable='dumpling',
        dev='master',
        version_kind='stable',
        adjust_repos=True,
    )

    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to install on',
    )

    parser.set_defaults(
        func=install,
    )

@priority(80)
def make_uninstall(parser):
    """
    Remove Ceph packages from remote hosts.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to uninstall Ceph from',
        )
    parser.set_defaults(
        func=uninstall,
        )

@priority(80)
def make_purge(parser):
    """
    Remove Ceph packages from remote hosts and purge all data.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph from',
        )
    parser.set_defaults(
        func=purge,
        )

@priority(80)
def make_purge_data(parser):
    """
    Purge (delete, destroy, discard, shred) any Ceph data from /var/lib/ceph
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph data from',
        )
    parser.set_defaults(
        func=purge_data,
        )
<|endoftext|>


<commit_before>import TileStache
import os
import shutil
import sys
import math
import json
from django.http import HttpResponse
from ModestMaps.Core import Coordinate
from ModestMaps.Geo import Location
from arches.app.datatypes import datatypes
from arches.app.datatypes.datatypes import DataTypeFactory
from arches.app.models import models
from arches.app.models.system_settings import settings
from arches.app.utils.geo_utils import GeoUtils

def get_tileserver_config(layer_id, request=None):
    database = settings.DATABASES['default']
    datatype_factory = DataTypeFactory()

    try:
        node = models.Node.objects.get(pk=layer_id)
        datatype = datatype_factory.get_instance(node.datatype)
        if request == None or request.user.has_perm('read_nodegroup', node.nodegroup):
            layer_config = datatype.get_layer_config(node)
        else:
            layer_config = datatype.get_layer_config(None)
    except Exception:
        layer_model = models.TileserverLayer.objects.get(name=layer_id)
        layer_config = layer_model.config

    tile_cache_config = settings.TILE_CACHE_CONFIG

    config_dict = {
        "cache": tile_cache_config,
        "layers": {}
    }
    config_dict["layers"][str(layer_id)] = layer_config
    return config_dict

def handle_request(request):
    path_info = request.path.replace('/tileserver/', '')
    layer_id = path_info.split('/')[0]
    config_dict = get_tileserver_config(layer_id, request=request)
    config = TileStache.Config.buildConfiguration(config_dict)
    query_string = ""
    script_name = ""
    response = HttpResponse()

    try:
        status_code, headers, content = TileStache.requestHandler2(
            config, path_info, query_string, script_name)

        response.content = content
        response.status_code = status_code
        for header, value in headers.items():
            response[header] = value

        response['Content-length'] = str(len(content))

    except Exception as e:
        print 'No tile data', e, response

    return response

def generateCoordinates(ul, lr, zooms, padding):
    count = 0

    for zoom in zooms:
        ul_ = ul.zoomTo(zoom).container().left(padding).up(padding)
        lr_ = lr.zoomTo(zoom).container().right(padding).down(padding)

        rows = lr_.row + 1 - ul_.row
        cols = lr_.column + 1 - ul_.column

        count += int(rows * cols)

    offset = 0
    for zoom in zooms:
        ul_ = ul.zoomTo(zoom).container().left(padding).up(padding)
        lr_ = lr.zoomTo(zoom).container().right(padding).down(padding)

        for row in range(int(ul_.row), int(lr_.row + 1)):
            for column in range(int(ul_.column), int(lr_.column + 1)):
                coord = Coordinate(row, column, zoom)

                yield (offset, count, coord)

                offset += 1

def clean_resource_cache(tile):
    # get the tile model's bounds
    datatype_factory = DataTypeFactory()
    nodegroup = models.NodeGroup.objects.get(pk=tile.nodegroup_id)
    for node in nodegroup.node_set.all():
        datatype = datatype_factory.get_instance(node.datatype)
        if datatype.should_cache(node) and datatype.should_manage_cache(node):
            bounds = datatype.get_bounds(tile, node)
            if bounds is not None:
                zooms = range(20)
                config = TileStache.parseConfig(
                    get_tileserver_config(node.nodeid))
                layer = config.layers[str(node.nodeid)]
                mimetype, format = layer.getTypeByExtension('pbf')

                lon1, lat1, lon2, lat2 = bounds
                south, west = min(lat1, lat2), min(lon1, lon2)
                north, east = max(lat1, lat2), max(lon1, lon2)

                northwest = Location(north, west)
                southeast = Location(south, east)

                ul = layer.projection.locationCoordinate(northwest)
                lr = layer.projection.locationCoordinate(southeast)

                padding = 0
                coordinates = generateCoordinates(ul, lr, zooms, padding)

                for (offset, count, coord) in coordinates:
                    config.cache.remove(layer, coord, format)
    for key, tile_list in tile.tiles.iteritems():
        for child_tile in tile_list:
            clean_resource_cache(child_tile)

def seed_resource_cache():
    datatype_factory = DataTypeFactory()
    zooms = range(settings.CACHE_SEED_MAX_ZOOM + 1)
    extension = 'pbf'

    lat1, lon1, lat2, lon2 = GeoUtils().get_bounds_from_geojson(settings.CACHE_SEED_BOUNDS)
    south, west = min(lat1, lat2), min(lon1, lon2)
    north, east = max(lat1, lat2), max(lon1, lon2)

    northwest = Location(north, west)
    southeast = Location(south, east)

    padding = 0

    datatypes = [
        d.pk for d in models.DDataType.objects.filter(isgeometric=True)]
    nodes = models.Node.objects.filter(
        graph__isresource=True, datatype__in=datatypes)
    for node in nodes:
        datatype = datatype_factory.get_instance(node.datatype)
        count = models.TileModel.objects.filter(
            data__has_key=str(node.nodeid)).count()
        if datatype.should_cache(node) and count > 0:
            config = TileStache.parseConfig(get_tileserver_config(node.nodeid))
            layer = config.layers[str(node.nodeid)]
            ul = layer.projection.locationCoordinate(northwest)
            lr = layer.projection.locationCoordinate(southeast)
            coordinates = generateCoordinates(ul, lr, zooms, padding)
            for (offset, count, coord) in coordinates:
                path = '%s/%d/%d/%d.%s' % (layer.name(), coord.zoom,
                                           coord.column, coord.row, extension)

                progress = {"tile": path,
                            "offset": offset + 1,
                            "total": count}

                attempts = 3
                rendered = False

                while not rendered:
                    print '%(offset)d of %(total)d...' % progress,

                    try:
                        mimetype, content = TileStache.getTile(
                            layer, coord, extension, True)

                    except:
                        attempts -= 1
                        print 'Failed %s, will try %s more.' % (progress['tile'], ['no', 'once', 'twice'][attempts])

                        if attempts == 0:
                            print 'Failed %(zoom)d/%(column)d/%(row)d, trying next tile.\n' % coord.__dict__
                            break

                    else:
                        rendered = True
                        progress['size'] = '%dKB' % (len(content) / 1024)

                        print '%(tile)s (%(size)s)' % progress
<commit_msg>parses tileserver url properly, re: #2925<commit_after>import TileStache
import os
import shutil
import sys
import math
import json
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from ModestMaps.Core import Coordinate
from ModestMaps.Geo import Location
from arches.app.datatypes import datatypes
from arches.app.datatypes.datatypes import DataTypeFactory
from arches.app.models import models
from arches.app.models.system_settings import settings
from arches.app.utils.geo_utils import GeoUtils

def get_tileserver_config(layer_id, request=None):
    database = settings.DATABASES['default']
    datatype_factory = DataTypeFactory()

    try:
        node = models.Node.objects.get(pk=layer_id)
        datatype = datatype_factory.get_instance(node.datatype)
        if request == None or request.user.has_perm('read_nodegroup', node.nodegroup):
            layer_config = datatype.get_layer_config(node)
        else:
            layer_config = datatype.get_layer_config(None)
    except Exception:
        layer_model = models.TileserverLayer.objects.get(name=layer_id)
        layer_config = layer_model.config

    tile_cache_config = settings.TILE_CACHE_CONFIG

    config_dict = {
        "cache": tile_cache_config,
        "layers": {}
    }
    config_dict["layers"][str(layer_id)] = layer_config
    return config_dict

def handle_request(request):
    path_info = request.path.replace(reverse('tileserver') + '/', '')
    layer_id = path_info.split('/')[0]
    config_dict = get_tileserver_config(layer_id, request=request)
    config = TileStache.Config.buildConfiguration(config_dict)
    query_string = ""
    script_name = ""
    response = HttpResponse()

    try:
        status_code, headers, content = TileStache.requestHandler2(
            config, path_info, query_string, script_name)

        response.content = content
        response.status_code = status_code
        for header, value in headers.items():
            response[header] = value

        response['Content-length'] = str(len(content))

    except Exception as e:
        print 'No tile data', e, response

    return response

def generateCoordinates(ul, lr, zooms, padding):
    count = 0

    for zoom in zooms:
        ul_ = ul.zoomTo(zoom).container().left(padding).up(padding)
        lr_ = lr.zoomTo(zoom).container().right(padding).down(padding)

        rows = lr_.row + 1 - ul_.row
        cols = lr_.column + 1 - ul_.column

        count += int(rows * cols)

    offset = 0
    for zoom in zooms:
        ul_ = ul.zoomTo(zoom).container().left(padding).up(padding)
        lr_ = lr.zoomTo(zoom).container().right(padding).down(padding)

        for row in range(int(ul_.row), int(lr_.row + 1)):
            for column in range(int(ul_.column), int(lr_.column + 1)):
                coord = Coordinate(row, column, zoom)

                yield (offset, count, coord)

                offset += 1

def clean_resource_cache(tile):
    # get the tile model's bounds
    datatype_factory = DataTypeFactory()
    nodegroup = models.NodeGroup.objects.get(pk=tile.nodegroup_id)
    for node in nodegroup.node_set.all():
        datatype = datatype_factory.get_instance(node.datatype)
        if datatype.should_cache(node) and datatype.should_manage_cache(node):
            bounds = datatype.get_bounds(tile, node)
            if bounds is not None:
                zooms = range(20)
                config = TileStache.parseConfig(
                    get_tileserver_config(node.nodeid))
                layer = config.layers[str(node.nodeid)]
                mimetype, format = layer.getTypeByExtension('pbf')

                lon1, lat1, lon2, lat2 = bounds
                south, west = min(lat1, lat2), min(lon1, lon2)
                north, east = max(lat1, lat2), max(lon1, lon2)

                northwest = Location(north, west)
                southeast = Location(south, east)

                ul = layer.projection.locationCoordinate(northwest)
                lr = layer.projection.locationCoordinate(southeast)

                padding = 0
                coordinates = generateCoordinates(ul, lr, zooms, padding)

                for (offset, count, coord) in coordinates:
                    config.cache.remove(layer, coord, format)
    for key, tile_list in tile.tiles.iteritems():
        for child_tile in tile_list:
            clean_resource_cache(child_tile)

def seed_resource_cache():
    datatype_factory = DataTypeFactory()
    zooms = range(settings.CACHE_SEED_MAX_ZOOM + 1)
    extension = 'pbf'

    lat1, lon1, lat2, lon2 = GeoUtils().get_bounds_from_geojson(settings.CACHE_SEED_BOUNDS)
    south, west = min(lat1, lat2), min(lon1, lon2)
    north, east = max(lat1, lat2), max(lon1, lon2)

    northwest = Location(north, west)
    southeast = Location(south, east)

    padding = 0

    datatypes = [
        d.pk for d in models.DDataType.objects.filter(isgeometric=True)]
    nodes = models.Node.objects.filter(
        graph__isresource=True, datatype__in=datatypes)
    for node in nodes:
        datatype = datatype_factory.get_instance(node.datatype)
        count = models.TileModel.objects.filter(
            data__has_key=str(node.nodeid)).count()
        if datatype.should_cache(node) and count > 0:
            config = TileStache.parseConfig(get_tileserver_config(node.nodeid))
            layer = config.layers[str(node.nodeid)]
            ul = layer.projection.locationCoordinate(northwest)
            lr = layer.projection.locationCoordinate(southeast)
            coordinates = generateCoordinates(ul, lr, zooms, padding)
            for (offset, count, coord) in coordinates:
                path = '%s/%d/%d/%d.%s' % (layer.name(), coord.zoom,
                                           coord.column, coord.row, extension)

                progress = {"tile": path,
                            "offset": offset + 1,
                            "total": count}

                attempts = 3
                rendered = False

                while not rendered:
                    print '%(offset)d of %(total)d...' % progress,

                    try:
                        mimetype, content = TileStache.getTile(
                            layer, coord, extension, True)

                    except:
                        attempts -= 1
                        print 'Failed %s, will try %s more.' % (progress['tile'], ['no', 'once', 'twice'][attempts])

                        if attempts == 0:
                            print 'Failed %(zoom)d/%(column)d/%(row)d, trying next tile.\n' % coord.__dict__
                            break

                    else:
                        rendered = True
                        progress['size'] = '%dKB' % (len(content) / 1024)

                        print '%(tile)s (%(size)s)' % progress
<|endoftext|>


<commit_before># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import logging
import subprocess
import shlex
import sys
import os
import tarfile
import zipfile
import tempfile
import time
import glob
import shutil
import hashlib

from cerbero.enums import Platform
from cerbero.utils import _, system_info, to_unixpath
from cerbero.utils import messages as m
from cerbero.errors import FatalError

PATCH = 'patch'
TAR = 'tar'

PLATFORM = system_info()[0]
LOGFILE = None  # open('/tmp/cerbero.log', 'w+')
DRY_RUN = False

def set_logfile_output(location):
    '''
    Sets a file to log

    @param location: path for the log file
    @type location: str
    '''
    if PLATFORM == Platform.WINDOWS:
        # silently return.
        return
    global LOGFILE
    if not LOGFILE is None:
        raise Exception("Logfile was already open. Forgot to call "
                        "close_logfile_output() ?")
    LOGFILE = open(location, "w+")

def close_logfile_output(dump=False):
    '''
    Close the current log file

    @param dump: dump the log file to stdout
    @type dump: bool
    '''
    if PLATFORM == Platform.WINDOWS:
        # silently return.
        return
    global LOGFILE
    if LOGFILE is None:
        raise Exception("No logfile was open")
    if dump:
        LOGFILE.seek(0)
        while True:
            data = LOGFILE.read()
            if data:
                print data
            else:
                break
    # if logfile is empty, remove it
    pos = LOGFILE.tell()
    LOGFILE.close()
    if pos == 0:
        os.remove(LOGFILE.name)
    LOGFILE = None

class StdOut:

    def __init__(self, stream=sys.stdout):
        self.stream = stream

    def write(self, data):
        self.stream.write(data)
        self.stream.flush()

    def __getattr__(self, attr):
        return getattr(self.stream, attr)

def _fix_mingw_cmd(path):
    reserved = ['/', ' ', '\\', ')', '(', '"']
    l_path = list(path)
    for i in range(len(path)):
        if path[i] == '\\':
            if i + 1 == len(path) or path[i + 1] not in reserved:
                l_path[i] = '/'
    return ''.join(l_path)

def call(cmd, cmd_dir='.', fail=True, verbose=False):
    '''
    Run a shell command

    @param cmd: the command to run
    @type cmd: str
    @param cmd_dir: directory where the command will be run
    @param cmd_dir: str
    @param fail: whether or not to raise an exception if the command fails
    @type fail: bool
    '''
    try:
        if LOGFILE is None:
            if verbose:
                m.message("Running command '%s'" % cmd)
        else:
            LOGFILE.write("Running command '%s'\n" % cmd)
        shell = True
        if PLATFORM == Platform.WINDOWS:
            # windows do not understand ./
            if cmd.startswith('./'):
                cmd = cmd[2:]
            # run all processes through sh.exe to get scripts working
            cmd = '%s "%s"' % ('sh -c', cmd)
            # fix paths with backslashes
            cmd = _fix_mingw_cmd(cmd)
            # Disable shell which uses cmd.exe
            shell = False
        stream = LOGFILE or sys.stdout
        if DRY_RUN:
            # write to sdterr so it's filtered more easilly
            m.error("cd %s && %s && cd %s" % (cmd_dir, cmd, os.getcwd()))
            ret = 0
        else:
            ret = subprocess.check_call(cmd, cwd=cmd_dir,
                                       stderr=subprocess.STDOUT,
                                       stdout=StdOut(stream),
                                       env=os.environ.copy(), shell=shell)
    except subprocess.CalledProcessError:
        if fail:
            raise FatalError(_("Error running command: %s") % cmd)
        else:
            ret = 0
    return ret

def check_call(cmd, cmd_dir=None, shell=False, split=True, fail=False):
    if split and isinstance(cmd, str):
        cmd = shlex.split(cmd)
    try:
        process = subprocess.Popen(cmd, cwd=cmd_dir,
                                   stdout=subprocess.PIPE,
                                   stderr=open(os.devnull), shell=shell)
        output, unused_err = process.communicate()
        if process.poll() and fail:
            raise Exception()
    except Exception:
        raise FatalError(_("Error running command: %s") % cmd)
    return output

def apply_patch(patch, directory, strip=1):
    '''
    Apply a patch

    @param patch: path of the patch file
    @type patch: str
    @param directory: directory to apply the apply
    @type: directory: str
    @param strip: strip
    @type strip: int
    '''

    logging.info("Applying patch %s" % (patch))
    call('%s -p%s -f -i %s' % (PATCH, strip, patch), directory)

def unpack(filepath, output_dir):
    '''
    Extracts a tarball

    @param filepath: path of the tarball
    @type filepath: str
    @param output_dir: output directory
    @type output_dir: str
    '''
    logging.info("Unpacking %s in %s" % (filepath, output_dir))
    if filepath.endswith('tar.gz') or filepath.endswith('tar.bz2') \
       or filepath.endswith('tbz2') or filepath.endswith('tgz'):
        tf = tarfile.open(filepath, mode='r:*')
        tf.extractall(path=output_dir)
    elif filepath.endswith('tar.xz'):
        call("%s -Jxf %s" % (TAR, to_unixpath(filepath)), output_dir)
    elif filepath.endswith('.zip'):
        zf = zipfile.ZipFile(filepath, "r")
        zf.extractall(path=output_dir)
    else:
        raise FatalError("Unknown tarball format %s" % filepath)

def download(url, destination=None, recursive=False, check_cert=True):
    '''
    Downloads a file with wget

    @param url: url to download
    @type: str
    @param destination: destination where the file will be saved
    @type destination: str
    '''
    cmd = "wget %s " % url
    path = None
    if recursive:
        cmd += "-r "
        path = destination
    else:
        if destination is not None:
            cmd += "-O %s " % destination

    if not check_cert:
        cmd += " --no-check-certificate"

    if not recursive and os.path.exists(destination):
        if LOGFILE is None:
            logging.info("File %s already downloaded." % destination)
    else:
        if not recursive and not os.path.exists(os.path.dirname(destination)):
            os.makedirs(os.path.dirname(destination))
        elif recursive and not os.path.exists(destination):
            os.makedirs(destination)

        if LOGFILE:
            LOGFILE.write("Downloading %s\n" % url)
        else:
            logging.info("Downloading %s", url)
        try:
            call(cmd, path)
        except FatalError, e:
            os.remove(destination)
            raise e

def download_curl(url, destination=None, recursive=False, check_cert=True):
    '''
    Downloads a file with cURL

    @param url: url to download
    @type: str
    @param destination: destination where the file will be saved
    @type destination: str
    '''
    path = None
    if recursive:
        raise FatalError(_("cURL doesn't support recursive downloads"))

    cmd = "curl -L "
    if not check_cert:
        cmd += " -k "
    if destination is not None:
        cmd += "%s -o %s " % (url, destination)
    else:
        cmd += "-O %s " % url

    if not recursive and os.path.exists(destination):
        logging.info("File %s already downloaded." % destination)
    else:
        if not recursive and not os.path.exists(os.path.dirname(destination)):
            os.makedirs(os.path.dirname(destination))
        elif recursive and not os.path.exists(destination):
            os.makedirs(destination)

        logging.info("Downloading %s", url)
        try:
            call(cmd, path)
        except FatalError, e:
            os.remove(destination)
            raise e

def _splitter(string, base_url):
    lines = string.split('\n')
    for line in lines:
        try:
            yield "%s/%s" % (base_url, line.split(' ')[2])
        except:
            continue

def recursive_download(url, destination):
    '''
    Recursive download for servers that don't return a list a url's but only
    the index.html file
    '''
    raw_list = check_call('curl %s' % url)

    with tempfile.NamedTemporaryFile() as f:
        for path in _splitter(raw_list, url):
            f.file.write(path + '\n')
        if not os.path.exists(destination):
            os.makedirs(destination)
        call("wget -i %s %s" % (f.name, url), destination)

def ls_files(files, prefix):
    if files == []:
        return files
    sfiles = check_call('ls %s' % ' '.join(files),
                        prefix, True, False, False).split('\n')
    sfiles.remove('')
    return list(set(sfiles))

def ls_dir(dirpath, prefix):
    files = []
    for root, dirnames, filenames in os.walk(dirpath):
        _root = root.split(prefix)[1]
        if _root[0] == '/':
            _root = _root[1:]
        files.extend([os.path.join(_root, x) for x in filenames])
    return files

def find_newer_files(prefix, compfile, include_link=False):
    include_links = include_link and '-L' or ''
    cmd = 'find %s * -type f -cnewer %s' % (include_links, compfile)
    sfiles = check_call(cmd, prefix, True, False, False).split('\n')
    sfiles.remove('')
    return sfiles

def replace(filepath, replacements):
    ''' Replaces keys in the 'replacements' dict with their values in file '''
    with open(filepath, 'r') as f:
        content = f.read()
    for k, v in replacements.iteritems():
        content = content.replace(k, v)
    with open(filepath, 'w+') as f:
        f.write(content)

def find_files(pattern, prefix):
    return glob.glob(os.path.join(prefix, pattern))

def prompt(message, options=[]):
    ''' Prompts the user for input with the message and options '''
    if len(options) != 0:
        message = "%s [%s] " % (message, '/'.join(options))
    res = raw_input(message)
    while res not in [str(x) for x in options]:
        res = raw_input(message)
    return res

def prompt_multiple(message, options):
    ''' Prompts the user for input with using a list of string options'''
    output = message + '\n'
    for i in range(len(options)):
        output += "[%s] %s\n" % (i, options[i])
    res = raw_input(output)
    while res not in [str(x) for x in range(len(options))]:
        res = raw_input(output)
    return options[int(res)]

def copy_dir(src, dest):
    if not os.path.exists(src):
        return
    for path in os.listdir(src):
        s = os.path.join(src, path)
        d = os.path.join(dest, path)
        if not os.path.exists(os.path.dirname(d)):
            os.makedirs(os.path.dirname(d))
        if os.path.isfile(s):
            shutil.copy(s, d)
        elif os.path.isdir(s):
            copy_dir(s, d)

def touch(path, create_if_not_exists=False, offset=0):
    if not os.path.exists(path):
        if create_if_not_exists:
            open(path, 'w').close()
        else:
            return
    t = time.time() + offset
    os.utime(path, (t, t))

def file_hash(path):
    '''
    Get the file md5 hash
    '''
    return hashlib.md5(open(path, 'rb').read()).digest()

def enter_build_environment(platform, arch, sourcedir=None):
    '''
    Enters to a new shell with the build environment
    '''
    BASHRC =  '''
if [ -e ~/.bashrc ]; then
source ~/.bashrc
fi
PS1='\[\033[01;32m\][cerbero-%s-%s]\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
'''

    bashrc = tempfile.NamedTemporaryFile()
    bashrc.write(BASHRC % (platform, arch))
    bashrc.flush()

    if sourcedir:
        os.chdir(sourcedir)

    if PLATFORM == Platform.WINDOWS:
        # $MINGW_PREFIX/home/username
        msys = os.path.join(os.path.expanduser('~'),
                            '..', '..', 'msys.bat')
        subprocess.check_call('%s -noxvrt' % msys)
    else:
        shell = os.environ.get('SHELL', '/bin/bash')
        if os.system("%s --rcfile %s -c echo 'test' > /dev/null 2>&1" % (shell, bashrc.name)) == 0:
            os.execlp(shell, shell, '--rcfile', bashrc.name)
        else:
            os.environ["CERBERO_ENV"] = "[cerbero-%s-%s]" % (platform, arch)
            os.execlp(shell, shell)

    bashrc.close()

def which(pgm, path=None):
    if path is None:
        path=os.getenv('PATH')
    for p in path.split(os.path.pathsep):
        p=os.path.join(p,pgm)
        if os.path.exists(p) and os.access(p,os.X_OK):
            return p
<commit_msg>Revert "shell: Don't ask too much from windows"<commit_after># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import logging
import subprocess
import shlex
import sys
import os
import tarfile
import zipfile
import tempfile
import time
import glob
import shutil
import hashlib

from cerbero.enums import Platform
from cerbero.utils import _, system_info, to_unixpath
from cerbero.utils import messages as m
from cerbero.errors import FatalError

PATCH = 'patch'
TAR = 'tar'

PLATFORM = system_info()[0]
LOGFILE = None  # open('/tmp/cerbero.log', 'w+')
DRY_RUN = False

def set_logfile_output(location):
    '''
    Sets a file to log

    @param location: path for the log file
    @type location: str
    '''

    global LOGFILE
    if not LOGFILE is None:
        raise Exception("Logfile was already open. Forgot to call "
                        "close_logfile_output() ?")
    LOGFILE = open(location, "w+")

def close_logfile_output(dump=False):
    '''
    Close the current log file

    @param dump: dump the log file to stdout
    @type dump: bool
    '''
    global LOGFILE
    if LOGFILE is None:
        raise Exception("No logfile was open")
    if dump:
        LOGFILE.seek(0)
        while True:
            data = LOGFILE.read()
            if data:
                print data
            else:
                break
    # if logfile is empty, remove it
    pos = LOGFILE.tell()
    LOGFILE.close()
    if pos == 0:
        os.remove(LOGFILE.name)
    LOGFILE = None

class StdOut:

    def __init__(self, stream=sys.stdout):
        self.stream = stream

    def write(self, data):
        self.stream.write(data)
        self.stream.flush()

    def __getattr__(self, attr):
        return getattr(self.stream, attr)

def _fix_mingw_cmd(path):
    reserved = ['/', ' ', '\\', ')', '(', '"']
    l_path = list(path)
    for i in range(len(path)):
        if path[i] == '\\':
            if i + 1 == len(path) or path[i + 1] not in reserved:
                l_path[i] = '/'
    return ''.join(l_path)

def call(cmd, cmd_dir='.', fail=True, verbose=False):
    '''
    Run a shell command

    @param cmd: the command to run
    @type cmd: str
    @param cmd_dir: directory where the command will be run
    @param cmd_dir: str
    @param fail: whether or not to raise an exception if the command fails
    @type fail: bool
    '''
    try:
        if LOGFILE is None:
            if verbose:
                m.message("Running command '%s'" % cmd)
        else:
            LOGFILE.write("Running command '%s'\n" % cmd)
        shell = True
        if PLATFORM == Platform.WINDOWS:
            # windows do not understand ./
            if cmd.startswith('./'):
                cmd = cmd[2:]
            # run all processes through sh.exe to get scripts working
            cmd = '%s "%s"' % ('sh -c', cmd)
            # fix paths with backslashes
            cmd = _fix_mingw_cmd(cmd)
            # Disable shell which uses cmd.exe
            shell = False
        stream = LOGFILE or sys.stdout
        if DRY_RUN:
            # write to sdterr so it's filtered more easilly
            m.error("cd %s && %s && cd %s" % (cmd_dir, cmd, os.getcwd()))
            ret = 0
        else:
            ret = subprocess.check_call(cmd, cwd=cmd_dir,
                                       stderr=subprocess.STDOUT,
                                       stdout=StdOut(stream),
                                       env=os.environ.copy(), shell=shell)
    except subprocess.CalledProcessError:
        if fail:
            raise FatalError(_("Error running command: %s") % cmd)
        else:
            ret = 0
    return ret

def check_call(cmd, cmd_dir=None, shell=False, split=True, fail=False):
    if split and isinstance(cmd, str):
        cmd = shlex.split(cmd)
    try:
        process = subprocess.Popen(cmd, cwd=cmd_dir,
                                   stdout=subprocess.PIPE,
                                   stderr=open(os.devnull), shell=shell)
        output, unused_err = process.communicate()
        if process.poll() and fail:
            raise Exception()
    except Exception:
        raise FatalError(_("Error running command: %s") % cmd)
    return output

def apply_patch(patch, directory, strip=1):
    '''
    Apply a patch

    @param patch: path of the patch file
    @type patch: str
    @param directory: directory to apply the apply
    @type: directory: str
    @param strip: strip
    @type strip: int
    '''

    logging.info("Applying patch %s" % (patch))
    call('%s -p%s -f -i %s' % (PATCH, strip, patch), directory)

def unpack(filepath, output_dir):
    '''
    Extracts a tarball

    @param filepath: path of the tarball
    @type filepath: str
    @param output_dir: output directory
    @type output_dir: str
    '''
    logging.info("Unpacking %s in %s" % (filepath, output_dir))
    if filepath.endswith('tar.gz') or filepath.endswith('tar.bz2') \
       or filepath.endswith('tbz2') or filepath.endswith('tgz'):
        tf = tarfile.open(filepath, mode='r:*')
        tf.extractall(path=output_dir)
    elif filepath.endswith('tar.xz'):
        call("%s -Jxf %s" % (TAR, to_unixpath(filepath)), output_dir)
    elif filepath.endswith('.zip'):
        zf = zipfile.ZipFile(filepath, "r")
        zf.extractall(path=output_dir)
    else:
        raise FatalError("Unknown tarball format %s" % filepath)

def download(url, destination=None, recursive=False, check_cert=True):
    '''
    Downloads a file with wget

    @param url: url to download
    @type: str
    @param destination: destination where the file will be saved
    @type destination: str
    '''
    cmd = "wget %s " % url
    path = None
    if recursive:
        cmd += "-r "
        path = destination
    else:
        if destination is not None:
            cmd += "-O %s " % destination

    if not check_cert:
        cmd += " --no-check-certificate"

    if not recursive and os.path.exists(destination):
        if LOGFILE is None:
            logging.info("File %s already downloaded." % destination)
    else:
        if not recursive and not os.path.exists(os.path.dirname(destination)):
            os.makedirs(os.path.dirname(destination))
        elif recursive and not os.path.exists(destination):
            os.makedirs(destination)

        if LOGFILE:
            LOGFILE.write("Downloading %s\n" % url)
        else:
            logging.info("Downloading %s", url)
        try:
            call(cmd, path)
        except FatalError, e:
            os.remove(destination)
            raise e

def download_curl(url, destination=None, recursive=False, check_cert=True):
    '''
    Downloads a file with cURL

    @param url: url to download
    @type: str
    @param destination: destination where the file will be saved
    @type destination: str
    '''
    path = None
    if recursive:
        raise FatalError(_("cURL doesn't support recursive downloads"))

    cmd = "curl -L "
    if not check_cert:
        cmd += " -k "
    if destination is not None:
        cmd += "%s -o %s " % (url, destination)
    else:
        cmd += "-O %s " % url

    if not recursive and os.path.exists(destination):
        logging.info("File %s already downloaded." % destination)
    else:
        if not recursive and not os.path.exists(os.path.dirname(destination)):
            os.makedirs(os.path.dirname(destination))
        elif recursive and not os.path.exists(destination):
            os.makedirs(destination)

        logging.info("Downloading %s", url)
        try:
            call(cmd, path)
        except FatalError, e:
            os.remove(destination)
            raise e

def _splitter(string, base_url):
    lines = string.split('\n')
    for line in lines:
        try:
            yield "%s/%s" % (base_url, line.split(' ')[2])
        except:
            continue

def recursive_download(url, destination):
    '''
    Recursive download for servers that don't return a list a url's but only
    the index.html file
    '''
    raw_list = check_call('curl %s' % url)

    with tempfile.NamedTemporaryFile() as f:
        for path in _splitter(raw_list, url):
            f.file.write(path + '\n')
        if not os.path.exists(destination):
            os.makedirs(destination)
        call("wget -i %s %s" % (f.name, url), destination)

def ls_files(files, prefix):
    if files == []:
        return files
    sfiles = check_call('ls %s' % ' '.join(files),
                        prefix, True, False, False).split('\n')
    sfiles.remove('')
    return list(set(sfiles))

def ls_dir(dirpath, prefix):
    files = []
    for root, dirnames, filenames in os.walk(dirpath):
        _root = root.split(prefix)[1]
        if _root[0] == '/':
            _root = _root[1:]
        files.extend([os.path.join(_root, x) for x in filenames])
    return files

def find_newer_files(prefix, compfile, include_link=False):
    include_links = include_link and '-L' or ''
    cmd = 'find %s * -type f -cnewer %s' % (include_links, compfile)
    sfiles = check_call(cmd, prefix, True, False, False).split('\n')
    sfiles.remove('')
    return sfiles

def replace(filepath, replacements):
    ''' Replaces keys in the 'replacements' dict with their values in file '''
    with open(filepath, 'r') as f:
        content = f.read()
    for k, v in replacements.iteritems():
        content = content.replace(k, v)
    with open(filepath, 'w+') as f:
        f.write(content)

def find_files(pattern, prefix):
    return glob.glob(os.path.join(prefix, pattern))

def prompt(message, options=[]):
    ''' Prompts the user for input with the message and options '''
    if len(options) != 0:
        message = "%s [%s] " % (message, '/'.join(options))
    res = raw_input(message)
    while res not in [str(x) for x in options]:
        res = raw_input(message)
    return res

def prompt_multiple(message, options):
    ''' Prompts the user for input with using a list of string options'''
    output = message + '\n'
    for i in range(len(options)):
        output += "[%s] %s\n" % (i, options[i])
    res = raw_input(output)
    while res not in [str(x) for x in range(len(options))]:
        res = raw_input(output)
    return options[int(res)]

def copy_dir(src, dest):
    if not os.path.exists(src):
        return
    for path in os.listdir(src):
        s = os.path.join(src, path)
        d = os.path.join(dest, path)
        if not os.path.exists(os.path.dirname(d)):
            os.makedirs(os.path.dirname(d))
        if os.path.isfile(s):
            shutil.copy(s, d)
        elif os.path.isdir(s):
            copy_dir(s, d)

def touch(path, create_if_not_exists=False, offset=0):
    if not os.path.exists(path):
        if create_if_not_exists:
            open(path, 'w').close()
        else:
            return
    t = time.time() + offset
    os.utime(path, (t, t))

def file_hash(path):
    '''
    Get the file md5 hash
    '''
    return hashlib.md5(open(path, 'rb').read()).digest()

def enter_build_environment(platform, arch, sourcedir=None):
    '''
    Enters to a new shell with the build environment
    '''
    BASHRC =  '''
if [ -e ~/.bashrc ]; then
source ~/.bashrc
fi
PS1='\[\033[01;32m\][cerbero-%s-%s]\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
'''

    bashrc = tempfile.NamedTemporaryFile()
    bashrc.write(BASHRC % (platform, arch))
    bashrc.flush()

    if sourcedir:
        os.chdir(sourcedir)

    if PLATFORM == Platform.WINDOWS:
        # $MINGW_PREFIX/home/username
        msys = os.path.join(os.path.expanduser('~'),
                            '..', '..', 'msys.bat')
        subprocess.check_call('%s -noxvrt' % msys)
    else:
        shell = os.environ.get('SHELL', '/bin/bash')
        if os.system("%s --rcfile %s -c echo 'test' > /dev/null 2>&1" % (shell, bashrc.name)) == 0:
            os.execlp(shell, shell, '--rcfile', bashrc.name)
        else:
            os.environ["CERBERO_ENV"] = "[cerbero-%s-%s]" % (platform, arch)
            os.execlp(shell, shell)

    bashrc.close()

def which(pgm, path=None):
    if path is None:
        path=os.getenv('PATH')
    for p in path.split(os.path.pathsep):
        p=os.path.join(p,pgm)
        if os.path.exists(p) and os.access(p,os.X_OK):
            return p
<|endoftext|>