<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-
"""
test top-level desisim functions
"""
#
from __future__ import absolute_import, division, print_function
#
import unittest
import re
from desisim import __version__ as theVersion
#
class TestTopLevel(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.versionre = re.compile(r'([0-9]+!)?([0-9]+)(\.[0-9]+)*((a|b|rc|\.post|\.dev)[0-9]+)?')

    @classmethod
    def tearDownClass(cls):
        pass

    def test_version(self):
        """Ensure the version conforms to PEP386/PEP440.
        """
        #- python 3.5 has assertRegexpMatches but has deprecated it in favor
        #- of assertRegexp, but that doesn't exist in python 2.7.  Keep
        #- assertRegexMatches for now until we've fully transitioned to py3.5
        self.assertRegex(theVersion,self.versionre)

if __name__ == '__main__':
    unittest.main()
<commit_msg>revert relative import<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-
"""
test top-level desisim functions
"""
#
from __future__ import absolute_import, division, print_function
#
import unittest
import re
from .. import __version__ as theVersion
#
class TestTopLevel(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.versionre = re.compile(r'([0-9]+!)?([0-9]+)(\.[0-9]+)*((a|b|rc|\.post|\.dev)[0-9]+)?')

    @classmethod
    def tearDownClass(cls):
        pass

    def test_version(self):
        """Ensure the version conforms to PEP386/PEP440.
        """
        #- python 3.5 has assertRegexpMatches but has deprecated it in favor
        #- of assertRegexp, but that doesn't exist in python 2.7.  Keep
        #- assertRegexMatches for now until we've fully transitioned to py3.5
        self.assertRegex(theVersion,self.versionre)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before><commit_msg>update<commit_after><|endoftext|>


<commit_before>import sys
import os
import errno
from zipfile import is_zipfile, ZipFile

import pycurl

import ee
import ee.mapclient

max_items = 99999

ee.Initialize()

def download_image_by_asset_path(asset_path, output_folder, overwrite=True):
	"""
		Downloads an individual image, given its asset path, and saves it to output_folder.
		Returns a list of the downloaded image or images
	"""
	### Get the download URL from EE
	image = ee.Image(asset_path)  # this approach comes from https://github.com/google/earthengine-api/blob/master/python/examples/py/Image/download.py
	path = image.getDownloadUrl({
		#'scale': 30,
		#'crs': 'EPSG:3310',
		#'region': '[[-120, 35], [-119, 35], [-119, 34], [-120, 34]]'
	})

	### Do some name management things
	output_name = os.path.split(asset_path)[1]
	zipfile = output_name + ".zip"
	download_path = os.path.join(output_folder, zipfile)  # comes as a zip file with a .tfw
	output_path = os.path.join(output_folder, output_name + ".tif")

	### Download the file
	with open(download_path, 'wb') as f:
		c = pycurl.Curl()
		c.setopt(c.URL, path)
		c.setopt(c.WRITEDATA, f)
		c.perform()
		c.close()

	### Extract the Zip and delete it
	if not is_zipfile(download_path):
		raise RuntimeError("Downloaded file was not a zip file!")

	with open(download_path, 'r') as zf:
		z = ZipFile(zf)
		downloaded_items = [os.path.join(output_folder,item) for item in z.namelist() if not item.endswith("tfw") ]
		z.extractall(path=output_folder)
		z.close()
		del z

	os.remove(download_path)

	return downloaded_items

def makedirs(path):
	"""
		Handles nice directory creation in Python 2.7 - see https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python/600612#600612
	"""
	try:
		os.makedirs(path)
	except OSError as exc:  # Python >2.5
		if exc.errno == errno.EEXIST and os.path.isdir(path):
			pass
		else:
			raise

def download_images_in_collection(collection_id, output_folder, max_items=max_items):
	"""
		Downloads images in ImageCollection specified by collection_id and saves them into
		the location specified in output folder, up to max_items downloads. Set max_items
		to a very high number to download all items.

		Returns a list with the full paths to the downloaded images.
	"""

	### Get all of the items in the collection
	collection = ee.ImageCollection(collection_id)
	collection_items = collection.toList(max_items).getInfo()

	# check that the output path exists - create it if not
	makedirs(output_folder)

	downloaded_items = []
	for item in collection_items:
		downloaded_items += download_image_by_asset_path(item["id"], output_folder)  # extend the list with the new list that's produced - don't append

	return downloaded_items

if __name__ == "__main__":
	args = sys.argv[1:]
	collection_id = args[0]
	output_folder = args[1]
	max_items = int(args[2])

	collection_id = "users/ucd-cws-ee-data/ssj-delta-cu/ssj-disalexi/eto_collection"  # for testing
	output_folder = "/mnt/c/Users/dsx/Code/ee-download/test"

	download__images_in_collection(collection_id, output_folder, max_items=max_items)<commit_msg>Removing demo paths<commit_after>import sys
import os
import errno
from zipfile import is_zipfile, ZipFile

import pycurl

import ee
import ee.mapclient

max_items = 99999

ee.Initialize()

def download_image_by_asset_path(asset_path, output_folder):
	"""
		Downloads an individual image, given its asset path, and saves it to output_folder.
		Returns a list of the downloaded image or images
	"""
	### Get the download URL from EE
	image = ee.Image(asset_path)  # this approach comes from https://github.com/google/earthengine-api/blob/master/python/examples/py/Image/download.py
	path = image.getDownloadUrl({
		#'scale': 30,
		#'crs': 'EPSG:3310',
		#'region': '[[-120, 35], [-119, 35], [-119, 34], [-120, 34]]'
	})

	### Do some name management things
	output_name = os.path.split(asset_path)[1]
	zipfile = output_name + ".zip"
	download_path = os.path.join(output_folder, zipfile)  # comes as a zip file with a .tfw
	output_path = os.path.join(output_folder, output_name + ".tif")

	### Download the file
	with open(download_path, 'wb') as f:
		c = pycurl.Curl()
		c.setopt(c.URL, path)
		c.setopt(c.WRITEDATA, f)
		c.perform()
		c.close()

	### Extract the Zip and delete it
	if not is_zipfile(download_path):
		raise RuntimeError("Downloaded file was not a zip file!")

	with open(download_path, 'r') as zf:
		z = ZipFile(zf)
		downloaded_items = [os.path.join(output_folder,item) for item in z.namelist() if not item.endswith("tfw") ]
		z.extractall(path=output_folder)
		z.close()
		del z

	os.remove(download_path)

	return downloaded_items

def makedirs(path):
	"""
		Handles nice directory creation in Python 2.7 - see https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python/600612#600612
	"""
	try:
		os.makedirs(path)
	except OSError as exc:  # Python >2.5
		if exc.errno == errno.EEXIST and os.path.isdir(path):
			pass
		else:
			raise

def download_images_in_collection(collection_id, output_folder, max_items=max_items):
	"""
		Downloads images in ImageCollection specified by collection_id and saves them into
		the location specified in output folder, up to max_items downloads. Set max_items
		to a very high number to download all items.

		Returns a list with the full paths to the downloaded images.
	"""

	### Get all of the items in the collection
	collection = ee.ImageCollection(collection_id)
	collection_items = collection.toList(max_items).getInfo()

	# check that the output path exists - create it if not
	makedirs(output_folder)

	downloaded_items = []
	for item in collection_items:
		downloaded_items += download_image_by_asset_path(item["id"], output_folder)  # extend the list with the new list that's produced - don't append

	return downloaded_items

if __name__ == "__main__":
	args = sys.argv[1:]
	collection_id = args[0]
	output_folder = args[1]
	max_items = int(args[2])

	download__images_in_collection(collection_id, output_folder, max_items=max_items)<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017-2018

import os
import re
import urllib

from pilot.user.atlas.setup import get_asetup
from pilot.user.atlas.setup import get_file_system_root_path
from pilot.info import infosys
from pilot.util.auxiliary import get_logger
from pilot.util.config import config
# import pilot.info.infoservice as infosys

import logging
logger = logging.getLogger(__name__)

def do_use_container(**kwargs):
    """
    Decide whether to use a container or not.

    :param kwargs: dictionary of key-word arguments.
    :return: True is function has decided that a container should be used, False otherwise (boolean).
    """

    use_container = True

    job = kwargs.get('job')
    if job:
        # for user jobs, TRF option --containerImage must have been used, ie imagename must be set
        if not (job.is_analysis() and job.imagename):
            use_container = False

    return use_container

def wrapper(executable, **kwargs):
    """
    Wrapper function for any container specific usage.
    This function will be called by pilot.util.container.execute() and prepends the executable with a container command.

    :param executable: command to be executed (string).
    :param kwargs: dictionary of key-word arguments.
    :return: executable wrapped with container command (string).
    """

    workdir = kwargs.get('workdir', '.')
    pilot_home = os.environ.get('PILOT_HOME', '')
    job = kwargs.get('job')

    logger.info('container wrapper called')

    if workdir == '.' and pilot_home != '':
        workdir = pilot_home

    # if job.imagename (from --containerimage <image>) is set, then always use raw singularity
    if config.Container.setup_type == "ALRB" and not job.imagename:
        fctn = alrb_wrapper
    else:
        fctn = singularity_wrapper
    return fctn(executable, workdir, job)

# def use_payload_container(job):
#     pass

def use_middleware_container():
    """
    Should middleware from container be used?
    In case middleware, i.e. the copy command for stage-in/out, should be taken from a container this function should
    return True.

    :return: True if middleware should be taken from container. False otherwise.
    """

    if get_middleware_type() == 'container':
        return True
    else:
        return False

def get_middleware_container():
    pass

def extract_platform_and_os(platform):
    """
    Extract the platform and OS substring from platform

    :param platform (string): E.g. "x86_64-slc6-gcc48-opt"
    :return: extracted platform specifics (string). E.g. "x86_64-slc6". In case of failure, return the full platform
    """

    pattern = r"([A-Za-z0-9_-]+)-.+-.+"
    a = re.findall(re.compile(pattern), platform)

    if len(a) > 0:
        ret = a[0]
    else:
        logger.warning("could not extract architecture and OS substring using pattern=%s from platform=%s"
                       "(will use %s for image name)" % (pattern, platform, platform))
        ret = platform

    return ret

def get_grid_image_for_singularity(platform):
    """
    Return the full path to the singularity grid image

    :param platform (string): E.g. "x86_64-slc6"
    :return: full path to grid image (string).
    """

    if not platform or platform == "":
        platform = "x86_64-slc6"
        logger.warning("using default platform=%s (cmtconfig not set)" % (platform))

    arch_and_os = extract_platform_and_os(platform)
    image = arch_and_os + ".img"
    _path = os.path.join(get_file_system_root_path(), "atlas.cern.ch/repo/containers/images/singularity")
    path = os.path.join(_path, image)
    if not os.path.exists(path):
        image = 'x86_64-centos7.img'
        logger.warning('path does not exist: %s (trying with image %s instead)' % (path, image))
        path = os.path.join(_path, image)
        if not os.path.exists(path):
            logger.warning('path does not exist either: %s' % path)
            path = ""

    return path

def get_middleware_type():
    """
    Return the middleware type from the container type.
    E.g. container_type = 'singularity:pilot;docker:wrapper;middleware:container'
    get_middleware_type() -> 'container', meaning that middleware should be taken from the container. The default
    is otherwise 'workernode', i.e. middleware is assumed to be present on the worker node.

    :return: middleware_type (string)
    """

    middleware_type = ""
    container_type = infosys.queuedata.container_type

    mw = 'middleware'
    if container_type and container_type != "" and mw in container_type:
        try:
            container_names = container_type.split(';')
            for name in container_names:
                t = name.split(':')
                if mw == t[0]:
                    middleware_type = t[1]
        except Exception as e:
            logger.warning("failed to parse the container name: %s, %s" % (container_type, e))
    else:
        # logger.warning("container middleware type not specified in queuedata")
        # no middleware type was specified, assume that middleware is present on worker node
        middleware_type = "workernode"

    return middleware_type

def alrb_wrapper(cmd, workdir, job):
    """
    Wrap the given command with the special ALRB setup for containers
    E.g. cmd = /bin/bash hello_world.sh
    ->
    export thePlatform="x86_64-slc6-gcc48-opt"
    export ALRB_CONT_RUNPAYLOAD="cmd'
    setupATLAS -c $thePlatform

    :param cmd (string): command to be executed in a container.
    :param workdir: (not used)
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    log = get_logger(job.jobid)
    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    if container_name == 'singularity':
        # first get the full setup, which should be removed from cmd (or ALRB setup won't work)
        _asetup = get_asetup()
        cmd = cmd.replace(_asetup, "asetup ")
        # get simplified ALRB setup (export)
        asetup = get_asetup(alrb=True)

        # Get the singularity options
        singularity_options = queuedata.container_options
        log.debug(
            "resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        _cmd = asetup
        if job.platform:
            _cmd += 'export thePlatform=\"%s\";' % job.platform
        #if '--containall' not in singularity_options:
        #    singularity_options += ' --containall'
        if singularity_options != "":
            _cmd += 'export ALRB_CONT_CMDOPTS=\"%s\";' % singularity_options
        _cmd += 'export ALRB_CONT_RUNPAYLOAD=\"%s\";' % cmd

        # this should not be necessary after the extract_container_image() in JobData update
        # containerImage should have been removed already
        if '--containerImage' in job.jobparams:
            job.jobparams, container_path = remove_container_string(job.jobparams)
            if container_path != "":
                _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c %s' % container_path
            else:
                log.warning('failed to extract container path from %s' % job.jobparams)
                _cmd = ""
        else:
            _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c images'
            if job.platform:
                _cmd += '+$thePlatform'

        _cmd = _cmd.replace('  ', ' ')
        cmd = _cmd
        log.info("Updated command: %s" % cmd)

    return cmd

## DEPRECATED, remove after verification with user container job
def remove_container_string(job_params):
    """ Retrieve the container string from the job parameters """

    pattern = r" \'?\-\-containerImage\=?\ ?([\S]+)\ ?\'?"
    compiled_pattern = re.compile(pattern)

    # remove any present ' around the option as well
    job_params = re.sub(r'\'\ \'', ' ', job_params)

    # extract the container path
    found = re.findall(compiled_pattern, job_params)
    container_path = found[0] if len(found) > 0 else ""

    # Remove the pattern and update the job parameters
    job_params = re.sub(pattern, ' ', job_params)

    return job_params, container_path

def singularity_wrapper(cmd, workdir, job):
    """
    Prepend the given command with the singularity execution command
    E.g. cmd = /bin/bash hello_world.sh
    -> singularity_command = singularity exec -B <bindmountsfromcatchall> <img> /bin/bash hello_world.sh
    singularity exec -B <bindmountsfromcatchall>  /cvmfs/atlas.cern.ch/repo/images/singularity/x86_64-slc6.img <script>

    :param cmd (string): command to be prepended.
    :param workdir: explicit work directory where the command should be executed (needs to be set for Singularity).
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    log = get_logger(job.jobid)
    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    log.debug("resolved container_name from queuedata.contaner_type: %s" % container_name)

    if container_name == 'singularity':
        log.info("singularity has been requested")

        # Get the singularity options
        singularity_options = queuedata.container_options + ",/cvmfs,${workdir},/home"
        log.debug("resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        if not singularity_options:
            log.warning('singularity options not set')

        # Get the image path
        if job.imagename:
            image_path = job.imagename
        else:
            image_path = get_grid_image_for_singularity(job.platform)

        # Does the image exist?
        if image_path != '':
            # Prepend it to the given command
            cmd = "export workdir=" + workdir + "; singularity exec " + singularity_options + " " + image_path + \
                  " /bin/bash -c \'cd $workdir;pwd;" + cmd.replace("\'", "\\'").replace('\"', '\\"') + "\'"

            # for testing user containers
            # singularity_options = "-B $PWD:/data --pwd / "
            # singularity_cmd = "singularity exec " + singularity_options + image_path
            # cmd = re.sub(r'-p "([A-Za-z0-9.%/]+)"', r'-p "%s\1"' % urllib.pathname2url(singularity_cmd), cmd)
        else:
            log.warning("singularity options found but image does not exist")

        log.info("Updated command: %s" % cmd)

    return cmd
<commit_msg>Removed unused import<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017-2018

import os
import re
# for user container test: import urllib

from pilot.user.atlas.setup import get_asetup
from pilot.user.atlas.setup import get_file_system_root_path
from pilot.info import infosys
from pilot.util.auxiliary import get_logger
from pilot.util.config import config
# import pilot.info.infoservice as infosys

import logging
logger = logging.getLogger(__name__)

def do_use_container(**kwargs):
    """
    Decide whether to use a container or not.

    :param kwargs: dictionary of key-word arguments.
    :return: True is function has decided that a container should be used, False otherwise (boolean).
    """

    use_container = True

    job = kwargs.get('job')
    if job:
        # for user jobs, TRF option --containerImage must have been used, ie imagename must be set
        if not (job.is_analysis() and job.imagename):
            use_container = False

    return use_container

def wrapper(executable, **kwargs):
    """
    Wrapper function for any container specific usage.
    This function will be called by pilot.util.container.execute() and prepends the executable with a container command.

    :param executable: command to be executed (string).
    :param kwargs: dictionary of key-word arguments.
    :return: executable wrapped with container command (string).
    """

    workdir = kwargs.get('workdir', '.')
    pilot_home = os.environ.get('PILOT_HOME', '')
    job = kwargs.get('job')

    logger.info('container wrapper called')

    if workdir == '.' and pilot_home != '':
        workdir = pilot_home

    # if job.imagename (from --containerimage <image>) is set, then always use raw singularity
    if config.Container.setup_type == "ALRB" and not job.imagename:
        fctn = alrb_wrapper
    else:
        fctn = singularity_wrapper
    return fctn(executable, workdir, job)

# def use_payload_container(job):
#     pass

def use_middleware_container():
    """
    Should middleware from container be used?
    In case middleware, i.e. the copy command for stage-in/out, should be taken from a container this function should
    return True.

    :return: True if middleware should be taken from container. False otherwise.
    """

    if get_middleware_type() == 'container':
        return True
    else:
        return False

def get_middleware_container():
    pass

def extract_platform_and_os(platform):
    """
    Extract the platform and OS substring from platform

    :param platform (string): E.g. "x86_64-slc6-gcc48-opt"
    :return: extracted platform specifics (string). E.g. "x86_64-slc6". In case of failure, return the full platform
    """

    pattern = r"([A-Za-z0-9_-]+)-.+-.+"
    a = re.findall(re.compile(pattern), platform)

    if len(a) > 0:
        ret = a[0]
    else:
        logger.warning("could not extract architecture and OS substring using pattern=%s from platform=%s"
                       "(will use %s for image name)" % (pattern, platform, platform))
        ret = platform

    return ret

def get_grid_image_for_singularity(platform):
    """
    Return the full path to the singularity grid image

    :param platform (string): E.g. "x86_64-slc6"
    :return: full path to grid image (string).
    """

    if not platform or platform == "":
        platform = "x86_64-slc6"
        logger.warning("using default platform=%s (cmtconfig not set)" % (platform))

    arch_and_os = extract_platform_and_os(platform)
    image = arch_and_os + ".img"
    _path = os.path.join(get_file_system_root_path(), "atlas.cern.ch/repo/containers/images/singularity")
    path = os.path.join(_path, image)
    if not os.path.exists(path):
        image = 'x86_64-centos7.img'
        logger.warning('path does not exist: %s (trying with image %s instead)' % (path, image))
        path = os.path.join(_path, image)
        if not os.path.exists(path):
            logger.warning('path does not exist either: %s' % path)
            path = ""

    return path

def get_middleware_type():
    """
    Return the middleware type from the container type.
    E.g. container_type = 'singularity:pilot;docker:wrapper;middleware:container'
    get_middleware_type() -> 'container', meaning that middleware should be taken from the container. The default
    is otherwise 'workernode', i.e. middleware is assumed to be present on the worker node.

    :return: middleware_type (string)
    """

    middleware_type = ""
    container_type = infosys.queuedata.container_type

    mw = 'middleware'
    if container_type and container_type != "" and mw in container_type:
        try:
            container_names = container_type.split(';')
            for name in container_names:
                t = name.split(':')
                if mw == t[0]:
                    middleware_type = t[1]
        except Exception as e:
            logger.warning("failed to parse the container name: %s, %s" % (container_type, e))
    else:
        # logger.warning("container middleware type not specified in queuedata")
        # no middleware type was specified, assume that middleware is present on worker node
        middleware_type = "workernode"

    return middleware_type

def alrb_wrapper(cmd, workdir, job):
    """
    Wrap the given command with the special ALRB setup for containers
    E.g. cmd = /bin/bash hello_world.sh
    ->
    export thePlatform="x86_64-slc6-gcc48-opt"
    export ALRB_CONT_RUNPAYLOAD="cmd'
    setupATLAS -c $thePlatform

    :param cmd (string): command to be executed in a container.
    :param workdir: (not used)
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    log = get_logger(job.jobid)
    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    if container_name == 'singularity':
        # first get the full setup, which should be removed from cmd (or ALRB setup won't work)
        _asetup = get_asetup()
        cmd = cmd.replace(_asetup, "asetup ")
        # get simplified ALRB setup (export)
        asetup = get_asetup(alrb=True)

        # Get the singularity options
        singularity_options = queuedata.container_options
        log.debug(
            "resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        _cmd = asetup
        if job.platform:
            _cmd += 'export thePlatform=\"%s\";' % job.platform
        #if '--containall' not in singularity_options:
        #    singularity_options += ' --containall'
        if singularity_options != "":
            _cmd += 'export ALRB_CONT_CMDOPTS=\"%s\";' % singularity_options
        _cmd += 'export ALRB_CONT_RUNPAYLOAD=\"%s\";' % cmd

        # this should not be necessary after the extract_container_image() in JobData update
        # containerImage should have been removed already
        if '--containerImage' in job.jobparams:
            job.jobparams, container_path = remove_container_string(job.jobparams)
            if container_path != "":
                _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c %s' % container_path
            else:
                log.warning('failed to extract container path from %s' % job.jobparams)
                _cmd = ""
        else:
            _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c images'
            if job.platform:
                _cmd += '+$thePlatform'

        _cmd = _cmd.replace('  ', ' ')
        cmd = _cmd
        log.info("Updated command: %s" % cmd)

    return cmd

## DEPRECATED, remove after verification with user container job
def remove_container_string(job_params):
    """ Retrieve the container string from the job parameters """

    pattern = r" \'?\-\-containerImage\=?\ ?([\S]+)\ ?\'?"
    compiled_pattern = re.compile(pattern)

    # remove any present ' around the option as well
    job_params = re.sub(r'\'\ \'', ' ', job_params)

    # extract the container path
    found = re.findall(compiled_pattern, job_params)
    container_path = found[0] if len(found) > 0 else ""

    # Remove the pattern and update the job parameters
    job_params = re.sub(pattern, ' ', job_params)

    return job_params, container_path

def singularity_wrapper(cmd, workdir, job):
    """
    Prepend the given command with the singularity execution command
    E.g. cmd = /bin/bash hello_world.sh
    -> singularity_command = singularity exec -B <bindmountsfromcatchall> <img> /bin/bash hello_world.sh
    singularity exec -B <bindmountsfromcatchall>  /cvmfs/atlas.cern.ch/repo/images/singularity/x86_64-slc6.img <script>

    :param cmd (string): command to be prepended.
    :param workdir: explicit work directory where the command should be executed (needs to be set for Singularity).
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    log = get_logger(job.jobid)
    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    log.debug("resolved container_name from queuedata.contaner_type: %s" % container_name)

    if container_name == 'singularity':
        log.info("singularity has been requested")

        # Get the singularity options
        singularity_options = queuedata.container_options + ",/cvmfs,${workdir},/home"
        log.debug("resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        if not singularity_options:
            log.warning('singularity options not set')

        # Get the image path
        if job.imagename:
            image_path = job.imagename
        else:
            image_path = get_grid_image_for_singularity(job.platform)

        # Does the image exist?
        if image_path != '':
            # Prepend it to the given command
            cmd = "export workdir=" + workdir + "; singularity exec " + singularity_options + " " + image_path + \
                  " /bin/bash -c \'cd $workdir;pwd;" + cmd.replace("\'", "\\'").replace('\"', '\\"') + "\'"

            # for testing user containers
            # singularity_options = "-B $PWD:/data --pwd / "
            # singularity_cmd = "singularity exec " + singularity_options + image_path
            # cmd = re.sub(r'-p "([A-Za-z0-9.%/]+)"', r'-p "%s\1"' % urllib.pathname2url(singularity_cmd), cmd)
        else:
            log.warning("singularity options found but image does not exist")

        log.info("Updated command: %s" % cmd)

    return cmd
<|endoftext|>


<commit_before>"""Dataset handling."""

# from functools import lru_cache
import logging

import numpy as np

from . import files, image, paths
from .types import ImageMode, Lesion, TextureSpec
from .util import unify_masks

class Dataset(object):
    """Dataset class for easier access to patients with images."""
    def __init__(self, mode, samplelist, cases=None):
        self.mode = ImageMode(mode)
        self.samplelist = samplelist
        self.cases = cases

    @property
    def samplelist_path(self):
        return paths.samplelist_path(self.mode, self.samplelist)

    def each_patient(self):
        """Generate patients."""
        patients = files.read_patients_file(self.samplelist_path)
        for p in patients:
            if self.cases is None or p.num in self.cases:
                yield p

    def each_image_id(self):
        for p in self.each_patient():
            for s in p.scans:
                yield p.num, s, p.lesions

    def each_lesion(self):
        """Generate lesions."""
        for case, scan, lesions in self.each_image_id():
            for lesion in lesions:
                yield case, scan, lesion

class DummyDataset(Dataset):
    def __init__(self, mode, samplelist, cases=None):
        self.mode = ImageMode(mode)
        self.samplelist = samplelist
        self.cases = cases

    @property
    def samplelist_path(self):
        raise NotImplementedError()

    def each_patient(self):
        raise NotImplementedError()

    def _cases(self):
        return range(300)

    def _scans(self):
        return (x + y for x in '12' for y in 'ab')

    def _lesions(self):
        return (Lesion(x, '0+0', 'NA') for x in range(3))

    def each_image_id(self):
        for case in self.cases or self._cases():
            for scan in self._scans():
                yield case, scan, list(self._lesions())

# @lru_cache(maxsize=16)
def read_prostate_mask(mode, case, scan):
    """Read prostate mask."""
    path = paths.mask_path(mode, 'prostate', case, scan)
    return image.Image.read_mask(path)

def read_lesion_mask(mode, case, scan, lesion):
    """Read lesion mask."""
    if isinstance(lesion, Lesion):
        lesion = lesion.index + 1
    path = paths.mask_path(mode, 'lesion', case, scan, lesion=lesion)
    return image.Image.read_mask(path)

# @lru_cache(maxsize=16)
def read_lesion_masks(mode, case, scan, lesions, only_largest=False):
    """Read and combine multiple lesion masks (for same scan)."""
    masks = (read_lesion_mask(mode, case, scan, x) for x in lesions)
    if only_largest:
        # Use only the biggest lesion.
        d = {np.count_nonzero(x): x for x in masks}
        masks = [d[max(d.keys())]]
        logging.warning([mode, case, scan, lesions, d.keys(),
                         np.count_nonzero(masks[0])])
    return unify_masks(masks)

def iterlesions(patients):
    """Generate all case, scan, lesion combinations."""
    if isinstance(patients, str):
        patients = files.read_patients_file(patients)
    for p in patients:
        for s in p.scans:
            for l in p.lesions:
                yield p, s, l

def read_tmap(mode, case, scan, tspec=None, masktype='prostate', **kwargs):
    """Read a texture map."""
    if tspec is None:
        tspec = TextureSpec('raw', 1, None)
    path = paths.texture_path(mode, case, scan, None, masktype, 'all', 0,
                              tspec, voxel='all')
    return image.Image.read(path, **kwargs)
<commit_msg>started ImageData class to iterate combinations of data identifiers<commit_after>"""Dataset handling."""

# from functools import lru_cache
import logging

import numpy as np

from . import files, image, paths
from .types import ImageMode, Lesion, TextureSpec
from .util import unify_masks

class Dataset(object):
    """Dataset class for easier access to patients with images."""
    def __init__(self, mode, samplelist, cases=None):
        self.mode = ImageMode(mode)
        self.samplelist = samplelist
        self.cases = cases

    @property
    def samplelist_path(self):
        return paths.samplelist_path(self.mode, self.samplelist)

    def each_patient(self):
        """Generate patients."""
        patients = files.read_patients_file(self.samplelist_path)
        for p in patients:
            if self.cases is None or p.num in self.cases:
                yield p

    def each_image_id(self):
        for p in self.each_patient():
            for s in p.scans:
                yield p.num, s, p.lesions

    def each_lesion(self):
        """Generate lesions."""
        for case, scan, lesions in self.each_image_id():
            for lesion in lesions:
                yield case, scan, lesion

class DummyDataset(Dataset):
    def __init__(self, mode, samplelist, cases=None):
        self.mode = ImageMode(mode)
        self.samplelist = samplelist
        self.cases = cases

    @property
    def samplelist_path(self):
        raise NotImplementedError()

    def each_patient(self):
        raise NotImplementedError()

    def _cases(self):
        return range(300)

    def _scans(self):
        return (x + y for x in '12' for y in 'ab')

    def _lesions(self):
        return (Lesion(x, '0+0', 'NA') for x in range(3))

    def each_image_id(self):
        for case in self.cases or self._cases():
            for scan in self._scans():
                yield case, scan, list(self._lesions())

# TODO: A new attempt!
class ImageData(object):
    """Generate and easily access all images, masks, gleason scores, etc.

    Combinations of values are indicated with lists of keys. Valid keys are
    mode, case, scan, lesion, masktype.
    """
    def __init__(self, modes, cases, scans=None, maxlesions=3, masktypes=None):
        if scans is None:
            scans = [x + y for x in '12' for y in 'ab']
        if masktypes is None:
            masktypes = ['prostate', 'lesion']
        self.choices = dict(
            mode=list(modes),
            case=list(cases),
            scan=list(scans),
            lesion=list(range(maxlesions)),
            masktype=list(masktypes),
        )

    @property
    def valid_keys(self):
        return list(self.choices.keys())

    def combinations(self, keys):
        """Generate all value combinations for `keys` (in given order).
        """
        yield from self._combinations(keys, {})

    def _combinations(self, keys, output):
        """Generate all value combinations for `keys` (in given order).

        Dictionary `output` is copied and passed recursively to keep state.
        """
        logging.debug([keys, output])
        if keys:
            key, rest = keys[0], keys[1:]
            values = self.choices[key]
            assert values, key
            for value in values:
                d = dict(output)
                d[key] = value
                yield from self._combinations(rest, d)
        else:
            yield output

# @lru_cache(maxsize=16)
def read_prostate_mask(mode, case, scan):
    """Read prostate mask."""
    path = paths.mask_path(mode, 'prostate', case, scan)
    return image.Image.read_mask(path)

def read_lesion_mask(mode, case, scan, lesion):
    """Read lesion mask."""
    if isinstance(lesion, Lesion):
        lesion = lesion.index + 1
    path = paths.mask_path(mode, 'lesion', case, scan, lesion=lesion)
    return image.Image.read_mask(path)

# @lru_cache(maxsize=16)
def read_lesion_masks(mode, case, scan, lesions, only_largest=False):
    """Read and combine multiple lesion masks (for same scan)."""
    masks = (read_lesion_mask(mode, case, scan, x) for x in lesions)
    if only_largest:
        # Use only the biggest lesion.
        d = {np.count_nonzero(x): x for x in masks}
        masks = [d[max(d.keys())]]
        logging.warning([mode, case, scan, lesions, d.keys(),
                         np.count_nonzero(masks[0])])
    return unify_masks(masks)

def iterlesions(patients):
    """Generate all case, scan, lesion combinations."""
    if isinstance(patients, str):
        patients = files.read_patients_file(patients)
    for p in patients:
        for s in p.scans:
            for l in p.lesions:
                yield p, s, l

def read_tmap(mode, case, scan, tspec=None, masktype='prostate', **kwargs):
    """Read a texture map."""
    if tspec is None:
        tspec = TextureSpec('raw', 1, None)
    path = paths.texture_path(mode, case, scan, None, masktype, 'all', 0,
                              tspec, voxel='all')
    return image.Image.read(path, **kwargs)
<|endoftext|>


<commit_before>import sys
from .. import command, utils, bar
import cairo

LEFT = object()
CENTER = object()
class _Drawer:
    """
        A helper class for drawing and text layout.
    """
    _fallbackFont = "-*-fixed-bold-r-normal-*-15-*-*-*-c-*-*-*"
    def __init__(self, qtile, window):
        self.qtile, self.window = qtile, window
        for i in qtile.conn.default_screen.allowed_depths:
            for v in i.visuals:
                if v.visual_id == qtile.conn.default_screen.root_visual:
                    break
        self.surface = cairo.XCBSurface(
                            qtile.conn.conn,
                            window.window.wid,
                            v,
                            window.width,
                            window.height
                        )
        self.set_background((0, 0, 1))

    def ctx(self):
        return cairo.Context(self.surface)

    def set_background(self, colour):
        c = self.ctx()
        c.set_source_rgb(*colour)
        c.rectangle(0, 0, self.window.width, self.window.height)
        c.fill()
        c.stroke()

    def textbox(self, text, x, y, width, height, padding = 0,
                alignment=LEFT, background=None, **attrs):
        """
            Draw text in the specified box using the current font. Text is
            centered vertically, and left-aligned.

            :background Fill box with the specified color first.
            :padding  Padding to the left of the text.
        """
        pass

    def rectangle(self, x, y, width, height, fillColor=None, borderColor=None, borderWidth=1):
        pass

class _Widget(command.CommandObject):
    """
        Each widget must set its own width attribute when the _configure method
        is called. If this is set to the special value bar.STRETCH, the bar itself
        will set the width to the maximum remaining space, after all other
        widgets have been configured. Only ONE widget per bar can have the
        bar.STRETCH width set.

        The offset attribute is set by the Bar after all widgets have been
        configured.
    """
    font = "-*-luxi mono-*-r-*-*-12-*-*-*-*-*-*-*"
    width = None
    offset = None
    name = None

    @property
    def win(self):
        return self.bar.window.window

    @property
    def colormap(self):
        return self.qtile.display.screen().default_colormap

    def _configure(self, qtile, bar, theme):
        self.qtile, self.bar, self.theme = qtile, bar, theme
        self._drawer = _Drawer(qtile, self.bar.window)

    def clear(self):
        self._drawer.rectangle(
            self.offset, 0, self.width, self.bar.size,
            self.bar.background
        )

    def info(self):
        return dict(
            name = self.__class__.__name__,
            offset = self.offset,
            width = self.width,
        )

    def click(self, x, y):
        pass

    def get(self, q, name):
        """
            Utility function for quick retrieval of a widget by name.
        """
        w = q.widgetMap.get(name)
        if not w:
            raise command.CommandError("No such widget: %s"%name)
        return w

    def _items(self, name):
        if name == "bar":
            return True, None

    def _select(self, name, sel):
        if name == "bar":
            return self.bar

    def cmd_info(self):
        """
            Info for this object.
        """
        return dict(name=self.name)

class _TextBox(_Widget):
    PADDING = 5
    def __init__(self, text=" ", width=bar.STRETCH):
        self.width = width
        self.text = text

    def _configure(self, qtile, bar, theme):
        _Widget._configure(self, qtile, bar, theme)
        if theme.font:
            self.font = theme.font

    def draw(self):
        self._drawer.textbox(
            self.text,
            self.offset, 0, self.width, self.bar.size,
            padding = self.PADDING,
            foreground=self.theme.fg_normal,
            background=self.theme.bg_normal,
        )

<commit_msg>Find root visual properly.<commit_after>import sys
from .. import command, utils, bar
import cairo

LEFT = object()
CENTER = object()
class _Drawer:
    """
        A helper class for drawing and text layout.
    """
    _fallbackFont = "-*-fixed-bold-r-normal-*-15-*-*-*-c-*-*-*"
    def __init__(self, qtile, window):
        self.qtile, self.window = qtile, window
        self.surface = cairo.XCBSurface(
                            qtile.conn.conn,
                            window.window.wid,
                            self.find_root_visual(),
                            window.width,
                            window.height
                        )
        self.set_background((0, 0, 1))

    def find_root_visual(self):
        for i in self.qtile.conn.default_screen.allowed_depths:
            for v in i.visuals:
                if v.visual_id == self.qtile.conn.default_screen.root_visual:
                    return v

    def ctx(self):
        return cairo.Context(self.surface)

    def set_background(self, colour):
        c = self.ctx()
        c.set_source_rgb(*colour)
        c.rectangle(0, 0, self.window.width, self.window.height)
        c.fill()
        c.stroke()

    def textbox(self, text, x, y, width, height, padding = 0,
                alignment=LEFT, background=None, **attrs):
        """
            Draw text in the specified box using the current font. Text is
            centered vertically, and left-aligned.

            :background Fill box with the specified color first.
            :padding  Padding to the left of the text.
        """
        pass

    def rectangle(self, x, y, width, height, fillColor=None, borderColor=None, borderWidth=1):
        pass

class _Widget(command.CommandObject):
    """
        Each widget must set its own width attribute when the _configure method
        is called. If this is set to the special value bar.STRETCH, the bar itself
        will set the width to the maximum remaining space, after all other
        widgets have been configured. Only ONE widget per bar can have the
        bar.STRETCH width set.

        The offset attribute is set by the Bar after all widgets have been
        configured.
    """
    font = "-*-luxi mono-*-r-*-*-12-*-*-*-*-*-*-*"
    width = None
    offset = None
    name = None

    @property
    def win(self):
        return self.bar.window.window

    @property
    def colormap(self):
        return self.qtile.display.screen().default_colormap

    def _configure(self, qtile, bar, theme):
        self.qtile, self.bar, self.theme = qtile, bar, theme
        self._drawer = _Drawer(qtile, self.bar.window)

    def clear(self):
        self._drawer.rectangle(
            self.offset, 0, self.width, self.bar.size,
            self.bar.background
        )

    def info(self):
        return dict(
            name = self.__class__.__name__,
            offset = self.offset,
            width = self.width,
        )

    def click(self, x, y):
        pass

    def get(self, q, name):
        """
            Utility function for quick retrieval of a widget by name.
        """
        w = q.widgetMap.get(name)
        if not w:
            raise command.CommandError("No such widget: %s"%name)
        return w

    def _items(self, name):
        if name == "bar":
            return True, None

    def _select(self, name, sel):
        if name == "bar":
            return self.bar

    def cmd_info(self):
        """
            Info for this object.
        """
        return dict(name=self.name)

class _TextBox(_Widget):
    PADDING = 5
    def __init__(self, text=" ", width=bar.STRETCH):
        self.width = width
        self.text = text

    def _configure(self, qtile, bar, theme):
        _Widget._configure(self, qtile, bar, theme)
        if theme.font:
            self.font = theme.font

    def draw(self):
        self._drawer.textbox(
            self.text,
            self.offset, 0, self.width, self.bar.size,
            padding = self.PADDING,
            foreground=self.theme.fg_normal,
            background=self.theme.bg_normal,
        )

<|endoftext|>


<commit_before>
import pytest

from astropy.tests.helper import assert_quantity_allclose
from astropy.units import allclose as quantity_allclose
from astropy import units as u
from astropy.coordinates import Longitude, Latitude, EarthLocation
from astropy.coordinates.sites import get_builtin_sites, get_downloaded_sites, SiteRegistry

def test_builtin_sites():
    reg = get_builtin_sites()

    greenwich = reg['greenwich']
    lon, lat, el = greenwich.to_geodetic()
    assert_quantity_allclose(lon, Longitude('0:0:0', unit=u.deg),
                             atol=10*u.arcsec)
    assert_quantity_allclose(lat, Latitude('51:28:40', unit=u.deg),
                             atol=1*u.arcsec)
    assert_quantity_allclose(el, 46*u.m, atol=1*u.m)

    names = reg.names
    assert 'greenwich' in names
    assert 'example_site' in names

    with pytest.raises(KeyError) as exc:
        reg['nonexistent site']
    assert exc.value.args[0] == "Site 'nonexistent site' not in database. Use the 'names' attribute to see available sites."

@pytest.mark.remote_data(source='astropy')
def test_online_sites():
    reg = get_downloaded_sites()

    keck = reg['keck']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    names = reg.names
    assert 'keck' in names
    assert 'ctio' in names

    with pytest.raises(KeyError) as exc:
        reg['nonexistent site']
    assert exc.value.args[0] == "Site 'nonexistent site' not in database. Use the 'names' attribute to see available sites."

    with pytest.raises(KeyError) as exc:
        reg['kec']
    assert exc.value.args[0] == "Site 'kec' not in database. Use the 'names' attribute to see available sites. Did you mean one of: 'keck'?'"

@pytest.mark.remote_data(source='astropy')
# this will *try* the online so we have to make it remote_data, even though it
# could fall back on the non-remote version
def test_EarthLocation_basic():
    greenwichel = EarthLocation.of_site('greenwich')
    lon, lat, el = greenwichel.to_geodetic()
    assert_quantity_allclose(lon, Longitude('0:0:0', unit=u.deg),
                             atol=10*u.arcsec)
    assert_quantity_allclose(lat, Latitude('51:28:40', unit=u.deg),
                             atol=1*u.arcsec)
    assert_quantity_allclose(el, 46*u.m, atol=1*u.m)

    names = EarthLocation.get_site_names()
    assert 'greenwich' in names
    assert 'example_site' in names

    with pytest.raises(KeyError) as exc:
        EarthLocation.of_site('nonexistent site')
    assert exc.value.args[0] == "Site 'nonexistent site' not in database. Use EarthLocation.get_site_names to see available sites."

def test_EarthLocation_state_offline():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_builtin=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_builtin=True)
    assert oldreg is not newreg

@pytest.mark.remote_data(source='astropy')
def test_EarthLocation_state_online():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_download=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_download=True)
    assert oldreg is not newreg

def test_registry():
    reg = SiteRegistry()

    assert len(reg.names) == 0

    names = ['sitea', 'site A']
    loc = EarthLocation.from_geodetic(lat=1*u.deg, lon=2*u.deg, height=3*u.km)
    reg.add_site(names, loc)

    assert len(reg.names) == 2

    loc1 = reg['SIteA']
    assert loc1 is loc

    loc2 = reg['sIte a']
    assert loc2 is loc

def test_non_EarthLocation():
    """
    A regression test for a typo bug pointed out at the bottom of
    https://github.com/astropy/astropy/pull/4042
    """
    class EarthLocation2(EarthLocation):
        pass

    # This lets keeps us from needing to do remote_data
    # note that this does *not* mess up the registry for EarthLocation because
    # registry is cached on a per-class basis
    EarthLocation2._get_site_registry(force_builtin=True)

    el2 = EarthLocation2.of_site('greenwich')
    assert type(el2) is EarthLocation2
    assert el2.info.name == 'Royal Observatory Greenwich'

def check_builtin_matches_remote(download_url=True):
    """
    This function checks that the builtin sites registry is consistent with the
    remote registry (or a registry at some other location).

    Note that current this is *not* run by the testing suite (because it
    doesn't start with "test", and is instead meant to be used as a check
    before merging changes in astropy-data)
    """
    builtin_registry = EarthLocation._get_site_registry(force_builtin=True)
    dl_registry = EarthLocation._get_site_registry(force_download=download_url)

    in_dl = {}
    matches = {}
    for name in builtin_registry.names:
        in_dl[name] = name in dl_registry
        if in_dl[name]:
            matches[name] = quantity_allclose(builtin_registry[name].geocentric, dl_registry[name].geocentric)
        else:
            matches[name] = False

    if not all(matches.values()):
        # this makes sure we actually see which don't match
        print("In builtin registry but not in download:")
        for name in in_dl:
            if not in_dl[name]:
                print('    ', name)
        print("In both but not the same value:")
        for name in matches:
            if not matches[name] and in_dl[name]:
                print('    ', name, 'builtin:', builtin_registry[name], 'download:', dl_registry[name])
        assert False, "Builtin and download registry aren't consistent - failures printed to stdout"

def test_meta_present():
    reg = get_builtin_sites()

    greenwich = reg['greenwich']
    assert greenwich.info.meta['source'] == ('Ordnance Survey via '
           'http://gpsinformation.net/main/greenwich.htm and UNESCO')
<commit_msg>Add a regression test for observatory name parsing<commit_after>
import pytest

from astropy.tests.helper import assert_quantity_allclose
from astropy.units import allclose as quantity_allclose
from astropy import units as u
from astropy.coordinates import Longitude, Latitude, EarthLocation
from astropy.coordinates.sites import get_builtin_sites, get_downloaded_sites, SiteRegistry

def test_builtin_sites():
    reg = get_builtin_sites()

    greenwich = reg['greenwich']
    lon, lat, el = greenwich.to_geodetic()
    assert_quantity_allclose(lon, Longitude('0:0:0', unit=u.deg),
                             atol=10*u.arcsec)
    assert_quantity_allclose(lat, Latitude('51:28:40', unit=u.deg),
                             atol=1*u.arcsec)
    assert_quantity_allclose(el, 46*u.m, atol=1*u.m)

    names = reg.names
    assert 'greenwich' in names
    assert 'example_site' in names

    with pytest.raises(KeyError) as exc:
        reg['nonexistent site']
    assert exc.value.args[0] == "Site 'nonexistent site' not in database. Use the 'names' attribute to see available sites."

@pytest.mark.remote_data(source='astropy')
def test_online_sites():
    reg = get_downloaded_sites()

    keck = reg['keck']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    names = reg.names
    assert 'keck' in names
    assert 'ctio' in names

    # The JSON file contains `name` and `aliases` for each site, and astropy
    # should use names from both, but not empty strings [#12721].
    assert '' not in names
    assert 'Royal Observatory Greenwich' in names

    with pytest.raises(KeyError) as exc:
        reg['nonexistent site']
    assert exc.value.args[0] == "Site 'nonexistent site' not in database. Use the 'names' attribute to see available sites."

    with pytest.raises(KeyError) as exc:
        reg['kec']
    assert exc.value.args[0] == "Site 'kec' not in database. Use the 'names' attribute to see available sites. Did you mean one of: 'keck'?'"

@pytest.mark.remote_data(source='astropy')
# this will *try* the online so we have to make it remote_data, even though it
# could fall back on the non-remote version
def test_EarthLocation_basic():
    greenwichel = EarthLocation.of_site('greenwich')
    lon, lat, el = greenwichel.to_geodetic()
    assert_quantity_allclose(lon, Longitude('0:0:0', unit=u.deg),
                             atol=10*u.arcsec)
    assert_quantity_allclose(lat, Latitude('51:28:40', unit=u.deg),
                             atol=1*u.arcsec)
    assert_quantity_allclose(el, 46*u.m, atol=1*u.m)

    names = EarthLocation.get_site_names()
    assert 'greenwich' in names
    assert 'example_site' in names

    with pytest.raises(KeyError) as exc:
        EarthLocation.of_site('nonexistent site')
    assert exc.value.args[0] == "Site 'nonexistent site' not in database. Use EarthLocation.get_site_names to see available sites."

def test_EarthLocation_state_offline():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_builtin=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_builtin=True)
    assert oldreg is not newreg

@pytest.mark.remote_data(source='astropy')
def test_EarthLocation_state_online():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_download=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_download=True)
    assert oldreg is not newreg

def test_registry():
    reg = SiteRegistry()

    assert len(reg.names) == 0

    names = ['sitea', 'site A']
    loc = EarthLocation.from_geodetic(lat=1*u.deg, lon=2*u.deg, height=3*u.km)
    reg.add_site(names, loc)

    assert len(reg.names) == 2

    loc1 = reg['SIteA']
    assert loc1 is loc

    loc2 = reg['sIte a']
    assert loc2 is loc

def test_non_EarthLocation():
    """
    A regression test for a typo bug pointed out at the bottom of
    https://github.com/astropy/astropy/pull/4042
    """
    class EarthLocation2(EarthLocation):
        pass

    # This lets keeps us from needing to do remote_data
    # note that this does *not* mess up the registry for EarthLocation because
    # registry is cached on a per-class basis
    EarthLocation2._get_site_registry(force_builtin=True)

    el2 = EarthLocation2.of_site('greenwich')
    assert type(el2) is EarthLocation2
    assert el2.info.name == 'Royal Observatory Greenwich'

def check_builtin_matches_remote(download_url=True):
    """
    This function checks that the builtin sites registry is consistent with the
    remote registry (or a registry at some other location).

    Note that current this is *not* run by the testing suite (because it
    doesn't start with "test", and is instead meant to be used as a check
    before merging changes in astropy-data)
    """
    builtin_registry = EarthLocation._get_site_registry(force_builtin=True)
    dl_registry = EarthLocation._get_site_registry(force_download=download_url)

    in_dl = {}
    matches = {}
    for name in builtin_registry.names:
        in_dl[name] = name in dl_registry
        if in_dl[name]:
            matches[name] = quantity_allclose(builtin_registry[name].geocentric, dl_registry[name].geocentric)
        else:
            matches[name] = False

    if not all(matches.values()):
        # this makes sure we actually see which don't match
        print("In builtin registry but not in download:")
        for name in in_dl:
            if not in_dl[name]:
                print('    ', name)
        print("In both but not the same value:")
        for name in matches:
            if not matches[name] and in_dl[name]:
                print('    ', name, 'builtin:', builtin_registry[name], 'download:', dl_registry[name])
        assert False, "Builtin and download registry aren't consistent - failures printed to stdout"

def test_meta_present():
    reg = get_builtin_sites()

    greenwich = reg['greenwich']
    assert greenwich.info.meta['source'] == ('Ordnance Survey via '
           'http://gpsinformation.net/main/greenwich.htm and UNESCO')
<|endoftext|>


<commit_before>#    Author: Sébastien BEAU <sebastien.beau@akretion.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
{
    "name": "Partner Helper",
    "version": "12.0.0.1.0",
    "author": "Akretion,Odoo Community Association (OCA)",
    "maintainer": "Akretion",
    "category": "Warehouse",
    "depends": [
        "base",
    ],
    "summary": "Add specific helper methods",
    "website": "https://github.com/OCA/partner-contact",
    "installable": True,
    "auto_install": False,
    "license": "AGPL-3",
}
<commit_msg>[MIG] partner_helper: Migration to 14.0<commit_after>#    Author: Sébastien BEAU <sebastien.beau@akretion.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
{
    "name": "Partner Helper",
    "version": "14.0.1.0.0",
    "author": "Akretion,Odoo Community Association (OCA)",
    "maintainer": "Akretion",
    "category": "Warehouse",
    "depends": [
        "base",
    ],
    "summary": "Add specific helper methods",
    "website": "https://github.com/OCA/partner-contact",
    "installable": True,
    "auto_install": False,
    "license": "AGPL-3",
}
<|endoftext|>


<commit_before>import json
import os
import sys

import pprint

pp = pprint.PrettyPrinter()

def configure_database(config, credentials):
    print("==Setting up Augur Database==")
    config['Database'] = {}
    config['Database']['database'] = credentials['database']
    config['Database']['host'] = credentials['host']
    config['Database']['port'] = credentials['port']
    config['Database']['user'] = credentials['db_user']
    config['Database']['password'] = credentials['password']
    config['Database']['schema'] = "augur_data"
    config['Database']['key'] = credentials['github_api_key']

    config['GitHub'] = {'apikey': credentials['github_api_key']}

def configure_defaults(config):
    print("==Setting up defaults==")

    if not 'Cache' in config:
        config['Cache'] = {}
        config['Cache']['config'] = {
            "cache.data_dir": "runtime/cache/",
            "cache.lock_dir": "runtime/cache/",
            "cache.type": "file"
        }

    if not 'Server' in config:
        config['Server'] = {
            "host": "0.0.0.0",
            "port": "5000",
            "workers": "4",
            "cache_expire": "3600"
        }

    if not 'Facade' in config:
        config["Facade"] = {
            "check_updates": 1,
            "clone_repos": 1,
            "create_xlsx_summary_files": 1,
            "delete_marked_repos": 0,
            "fix_affiliations": 1,
            "force_analysis": 1,
            "force_invalidate_caches": 1,
            "force_updates": 1,
            "limited_run": 0,
            "multithreaded": 0,
            "nuke_stored_affiliations": 0,
            "pull_repos": 1,
            "rebuild_caches": 1,
            "run_analysis": 1
        }
        print("Set default values for Facade...")

    if not 'Development' in config:
        config["Development"] = {
            "developer": "0",
            "interactive": "0"
        }
        print("Set default values for Developement...")

    if not 'Plugins' in config:
        config['Plugins'] = []
        print("Set default values for Plugins...")

    if not 'Housekeeper' in config:
        config['Housekeeper'] = {
            "jobs": [
                {
                    "delay": 150000,
                    "given": ["github_url"],
                    "model": "issues",
                    "repo_group_id": 0,
                    "all_focused": 1
                },
                {
                    "model": "repo_info",
                    "given": ["github_url"],
                    "delay": 150000,
                    "repo_group_id": 0
                },
                {
                    "model": "commits",
                    "given": ["repo_group"],
                    "delay": 150000,
                    "repo_group_id": 0
                },
                {
                    "model": "pull_requests",
                    "given": ["github_url"],
                    "delay": 1000000,
                    "repo_group_id": 0
                },
                {
                    "delay": 1000000,
                    "given": ["github_url"],
                    "model": "contributors",
                    "repo_group_id": 0
                },
                {
                    "delay": 1000000,
                    "given": ["git_url"],
                    "model": "insights",
                    "repo_group_id": 0
                }
            ]
        }
        print("Set default values for Housekeeper...")

def configure_workers(config, credentials):

    if credentials['facade_repo_path'][-1] != '/':
        credentials['facade_repo_path'] += '/'

    if not 'Workers' in config:
        config['Workers'] = {
            "facade_worker": {
                "port": 56111,
                "repo_directory": credentials['facade_repo_path'],
                "switch": 0,
                "workers": 1
            },
            "github_worker": {
                "port": 56211,
                "switch": 0,
                "workers": 2
            },
            "insight_worker": {
                "port": 56311,
                "switch": 0,
                "workers": 1
            },
            "pull_request_worker": {
                "port": 56411,
                "switch": 0,
                "workers": 1
            },
            "repo_info_worker": {
                "port": 56511,
                "switch": 0,
                "workers": 1
            },
            "value_worker": {
                "port": 56611,
                "scc_bin": "scc",
                "switch": 0,
                "workers": 1
            },
            "metric_status_worker": {
                "port": 56711,
                "switch": 0,
                "workers": 1
            },
            "linux_badge_worker": {
                "port": 56811,
                "switch": 0,
                "workers": 1
            },
            "license_worker": {
                    "port": 51242,
                    "switch": 0,
                    "workers": 1,
                "tagfile": "3.0.tag"
            }
        }
        print("Set default values for Workers")

    print()

def main():
    print("Beginning 'augur.config.json' creation process...\n")
    config = {}

    with open('temp.config.json', 'r') as db_credentials_file:
        credentials = json.load(db_credentials_file)
        configure_database(config, credentials)
        configure_defaults(config)
        configure_workers(config, credentials)

    try:
        with open('../../../augur.config.json', 'w') as f:
            f.write(json.dumps(config, indent=4))
            print('augur.config.json successfully created')
    except Exception as e:
        print("Error writing augur.config.json " + str(e))

if __name__ == "__main__":
    main()
<commit_msg>Add badge worker block to install-generated config<commit_after>import json
import os
import sys

import pprint

pp = pprint.PrettyPrinter()

def configure_database(config, credentials):
    print("==Setting up Augur Database==")
    config['Database'] = {}
    config['Database']['database'] = credentials['database']
    config['Database']['host'] = credentials['host']
    config['Database']['port'] = credentials['port']
    config['Database']['user'] = credentials['db_user']
    config['Database']['password'] = credentials['password']
    config['Database']['schema'] = "augur_data"
    config['Database']['key'] = credentials['github_api_key']

    config['GitHub'] = {'apikey': credentials['github_api_key']}

def configure_defaults(config):
    print("==Setting up defaults==")

    if not 'Cache' in config:
        config['Cache'] = {}
        config['Cache']['config'] = {
            "cache.data_dir": "runtime/cache/",
            "cache.lock_dir": "runtime/cache/",
            "cache.type": "file"
        }

    if not 'Server' in config:
        config['Server'] = {
            "host": "0.0.0.0",
            "port": "5000",
            "workers": "4",
            "cache_expire": "3600"
        }

    if not 'Facade' in config:
        config["Facade"] = {
            "check_updates": 1,
            "clone_repos": 1,
            "create_xlsx_summary_files": 1,
            "delete_marked_repos": 0,
            "fix_affiliations": 1,
            "force_analysis": 1,
            "force_invalidate_caches": 1,
            "force_updates": 1,
            "limited_run": 0,
            "multithreaded": 0,
            "nuke_stored_affiliations": 0,
            "pull_repos": 1,
            "rebuild_caches": 1,
            "run_analysis": 1
        }
        print("Set default values for Facade...")

    if not 'Development' in config:
        config["Development"] = {
            "developer": "0",
            "interactive": "0"
        }
        print("Set default values for Developement...")

    if not 'Plugins' in config:
        config['Plugins'] = []
        print("Set default values for Plugins...")

    if not 'Housekeeper' in config:
        config['Housekeeper'] = {
            "jobs": [
                {
                "all_focused": 1,
                "delay": 150000,
                "given": [
                    "github_url"
                ],
                "model": "issues",
                "repo_group_id": 0
                },
                {
                    "delay": 150000,
                    "given": [
                        "github_url"
                    ],
                    "model": "repo_info",
                    "repo_group_id": 0
                },
                {
                    "delay": 150000,
                    "given": [
                        "repo_group"
                    ],
                    "model": "commits",
                    "repo_group_id": 0
                },
                {
                    "delay": 1000000,
                    "given": [
                        "github_url"
                    ],
                    "model": "pull_requests",
                    "repo_group_id": 0
                },
                {
                    "delay": 1000000,
                    "given": [
                        "github_url"
                    ],
                    "model": "contributors",
                    "repo_group_id": 0
                },
                {
                    "delay": 1000000,
                    "given": [
                        "git_url"
                    ],
                    "model": "insights",
                    "repo_group_id": 0
                },
                {
                    "delay": 1000000,
                    "given": [
                        "git_url"
                    ],
                    "model": "badges",
                    "repo_group_id": 0
                }
            ]
        }
        print("Set default values for Housekeeper...")

def configure_workers(config, credentials):

    if credentials['facade_repo_path'][-1] != '/':
        credentials['facade_repo_path'] += '/'

    if not 'Workers' in config:
        config['Workers'] = {
            "facade_worker": {
                "port": 56111,
                "repo_directory": credentials['facade_repo_path'],
                "switch": 0,
                "workers": 1
            },
            "github_worker": {
                "port": 56211,
                "switch": 0,
                "workers": 2
            },
            "insight_worker": {
                "port": 56311,
                "switch": 0,
                "workers": 1
            },
            "pull_request_worker": {
                "port": 56411,
                "switch": 0,
                "workers": 1
            },
            "repo_info_worker": {
                "port": 56511,
                "switch": 0,
                "workers": 1
            },
            "value_worker": {
                "port": 56611,
                "scc_bin": "scc",
                "switch": 0,
                "workers": 1
            },
            "metric_status_worker": {
                "port": 56711,
                "switch": 0,
                "workers": 1
            },
            "linux_badge_worker": {
                "port": 56811,
                "switch": 0,
                "workers": 1
            },
            "license_worker": {
                    "port": 51242,
                    "switch": 0,
                    "workers": 1,
                "tagfile": "3.0.tag"
            }
        }
        print("Set default values for Workers")

    print()

def main():
    print("Beginning 'augur.config.json' creation process...\n")
    config = {}

    with open('temp.config.json', 'r') as db_credentials_file:
        credentials = json.load(db_credentials_file)
        configure_database(config, credentials)
        configure_defaults(config)
        configure_workers(config, credentials)

    try:
        with open('../../../augur.config.json', 'w') as f:
            f.write(json.dumps(config, indent=4))
            print('augur.config.json successfully created')
    except Exception as e:
        print("Error writing augur.config.json " + str(e))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""settings.py

Udacity conference server-side Python App Engine app user settings

$Id$

created/forked from conference.py by wesc on 2014 may 24

"""

# Replace the following lines with client IDs obtained from the APIs
# Console or Cloud Console.
WEB_CLIENT_ID = '42309308242-q33sjm01tqv8b4qs94bvnkphmgpdm5tb.apps.googleusercontent.com'

<commit_msg>test on local host and appspot<commit_after>#!/usr/bin/env python

"""settings.py

"""

# Replace the following lines with client IDs obtained from the APIs
# Console or Cloud Console.
WEB_CLIENT_ID = '42309308242-q33sjm01tqv8b4qs94bvnkphmgpdm5tb.apps.googleusercontent.com'

<|endoftext|>


<commit_before>#!python
# coding=utf-8

import os
import random
import bisect
import calendar
from datetime import datetime

import netCDF4
import numpy as np
import pandas as pd

from pyaxiom import logger

class Profile(object):

    def __init__(self, df=None, global_attributes=None, variable_attributes=None, fill_value=None, vertical_positive=None, base_time=None):

        self.df = df if isinstance(df, pd.DataFrame) and not df.empty else pd.DataFrame()
        self.fill_value = fill_value or np.dtype('f8').type(-9999.9)
        self.global_attributes = global_attributes or {}
        self.variable_attributes = variable_attributes or {}
        self.vertical_positive = vertical_positive or 'down'
        self.base_time = base_time or 'seconds since 1970-01-01 00:00:00'

    @property
    def variable_attributes(self):
        defaults = {
            'time' : {
                'units' : self.base_time,
                'standard_name' : 'time',
                'long_name': 'time'
            },
            'latitude' : {
                'units' : 'degrees_north',
                'standard_name' : 'latitude',
                'long_name' : 'latitude',
                'axis': 'Y'
            },
            'longitude' : {
                'units' : 'degrees_east',
                'standard_name' : 'longitude',
                'long_name' : 'longitude',
                'axis': 'X'
            },
            'z' : {
                'units' : 'm',
                'standard_name' : 'depth',
                'long_name' : 'depth',
                'positive': self.vertical_positive,
                'axis': 'Z'
            },
            'profile' : {
                'cf_role' : 'profile_id'
            },
            'crs' : {
                'long_name' : 'http://www.opengis.net/def/crs/EPSG/0/4326',
                'grid_mapping_name' : 'latitude_longitude',
                'epsg_code' : 'EPSG:4326',
                'semi_major_axis' : float(6378137.0),
                'inverse_flattening' : float(298.257223563)
            },
            'platform' : {
                'definition' : "http://mmisw.org/ont/ioos/definition/stationID"
            }
        }

        defaults.update(self._variable_attributes)
        return defaults

    @variable_attributes.setter
    def variable_attributes(self, vas):
        self._variable_attributes = vas

    @property
    def global_attributes(self):
        gas = self._global_attributes
        gas.update({
           'geospatial_vertical_positive': self.vertical_positive,
           'date_created': datetime.utcnow().strftime("%Y-%m-%dT%H:%M:00Z"),
           'Conventions': 'CF-1.6',
           'Metadata_conventions': 'Unidata Dataset Discovery v1.0',
           'featureType': 'profile',
           'cdm_data_type': 'Profile'
        })

        if not self.df.empty:
            # Time
            starting = self.df['time'].min()
            ending = self.df['time'].max()
            duration = "P%sS" % str(int(round((ending - starting).total_seconds())))
            gas.update({
                'time_coverage_start': starting.strftime("%Y-%m-%dT%H:%M:00Z"),
                'time_coverage_end': ending.strftime("%Y-%m-%dT%H:%M:00Z"),
                'time_coverage_duration': duration,
            })
            diffs = self.df['time'].unique()[1:] - self.df['time'].unique()[:-1]
            uniqs, inverse = np.unique(diffs, return_inverse=True)
            if uniqs.size > 1:
                time_diffs = diffs[np.bincount(inverse).argmax()]
                gas.update({
                    'time_coverage_resolution': "P%sS" % str(round(time_diffs.astype('timedelta64[s]').astype(int)))
                })

            # Vertical
            gas.update({
                'geospatial_vertical_min': self.df['z'].min(),
                'geospatial_vertical_max': self.df['z'].max(),
            })

            # Horizontal
            gas.update({
                'geospatial_lat_min': self.df['latitude'].min(),
                'geospatial_lat_max': self.df['latitude'].max(),
                'geospatial_lon_min': self.df['longitude'].min(),
                'geospatial_lon_max': self.df['longitude'].max(),
            })

        return gas

    @global_attributes.setter
    def global_attributes(self, gas):
        # These are set by this script, we don't someone to be able to set them manually
        global_skips = ["time_coverage_start", "time_coverage_end", "time_coverage_duration", "time_coverage_resolution",
                        "featureType", "geospatial_vertical_positive", "geospatial_vertical_min", "geospatial_vertical_max",
                        "geospatial_lat_min", "geospatial_lon_min", "geospatial_lat_max", "geospatial_lon_max",
                        "Conventions", "date_created", "cdm_data_type"]

        for i in set(global_skips) & gas.keys():
            logger.warning("Ignoring global attribute {} because it is calculated or set automatically".format(i))

        self._global_attributes = { k: v for k, v in gas.items() if k not in global_skips }

    def export(self, output_file, file_type=None):
        # Make output directory
        if not os.path.exists(os.path.dirname(output_file)):
            os.makedirs(os.path.dirname(output_file))

class IncompleteProfile(Profile):

    def export(self, output_file):
        super(IncompleteProfile, self).export(output_file)

        with netCDF4.Dataset(output_file, 'w', clobber=True) as nc:

            gas = self.global_attributes
            nc.setncatts(gas)

            profiles = self.df.profile.unique().size
            profile_group = self.df.groupby('profile')
            max_z = profile_group.size().max()

            nc.createDimension('profile', profiles)
            nc.createDimension('z', max_z)

            profile = nc.createVariable('profile', self.df.profile.dtype, ('profile',))
            _, unique_profile_rows = np.unique(self.df.profile.values, return_index=True)
            profile[:] = list(range(profiles))

            time = nc.createVariable('time', int, ('profile',))
            time[:] = netCDF4.date2num([datetime.utcfromtimestamp(t) for t in self.df.time.unique().astype('<M8[s]').astype(int)], units=self.base_time)

            latitude = nc.createVariable('latitude', self.df.latitude.dtype, ('profile',))
            latitude[:] = self.df.latitude.values[unique_profile_rows]

            longitude = nc.createVariable('longitude', self.df.longitude.dtype, ('profile',))
            longitude[:] = self.df.longitude.values[unique_profile_rows]

            # Metadata variables
            nc.createVariable("crs", 'i4')
            nc.createVariable("platform", "i4")
            nc.setncattr('platform', 'platform')

            # Data vars
            reserved_columns = ['profile', 'time', 'latitude', 'longitude']
            for i, (name, p) in enumerate(profile_group):
                for c in [d for d in self.df.columns if d not in reserved_columns]:
                    var_name = c.split(' ')[0].lower()
                    fill = self.fill_value.astype(p[c].dtype)
                    if var_name not in nc.variables:
                        v = nc.createVariable(var_name, self.df[c].dtype, ('profile', 'z'), fill_value=fill)
                    else:
                        v = nc.variables[var_name]
                    assignable_values = p[c].fillna(fill).values
                    v[i, :len(assignable_values)] = assignable_values

            for k, v in self.variable_attributes.items():
                if k in nc.variables:
                    for n, z in v.items():
                        try:
                            nc.variables[k].setncattr(n, z)
                        except BaseException:
                            logger.warning('Could not set attribute {} on {}'.format(n, k))
<commit_msg>Fix whitespace<commit_after>#!python
# coding=utf-8

import os
import random
import bisect
import calendar
from datetime import datetime

import netCDF4
import numpy as np
import pandas as pd

from pyaxiom import logger

class Profile(object):

    def __init__(self, df=None, global_attributes=None, variable_attributes=None, fill_value=None, vertical_positive=None, base_time=None):

        self.df = df if isinstance(df, pd.DataFrame) and not df.empty else pd.DataFrame()
        self.fill_value = fill_value or np.dtype('f8').type(-9999.9)
        self.global_attributes = global_attributes or {}
        self.variable_attributes = variable_attributes or {}
        self.vertical_positive = vertical_positive or 'down'
        self.base_time = base_time or 'seconds since 1970-01-01 00:00:00'

    @property
    def variable_attributes(self):
        defaults = {
            'time' : {
                'units' : self.base_time,
                'standard_name' : 'time',
                'long_name': 'time'
            },
            'latitude' : {
                'units' : 'degrees_north',
                'standard_name' : 'latitude',
                'long_name' : 'latitude',
                'axis': 'Y'
            },
            'longitude' : {
                'units' : 'degrees_east',
                'standard_name' : 'longitude',
                'long_name' : 'longitude',
                'axis': 'X'
            },
            'z' : {
                'units' : 'm',
                'standard_name' : 'depth',
                'long_name' : 'depth',
                'positive': self.vertical_positive,
                'axis': 'Z'
            },
            'profile' : {
                'cf_role' : 'profile_id'
            },
            'crs' : {
                'long_name' : 'http://www.opengis.net/def/crs/EPSG/0/4326',
                'grid_mapping_name' : 'latitude_longitude',
                'epsg_code' : 'EPSG:4326',
                'semi_major_axis' : float(6378137.0),
                'inverse_flattening' : float(298.257223563)
            },
            'platform' : {
                'definition' : "http://mmisw.org/ont/ioos/definition/stationID"
            }
        }

        defaults.update(self._variable_attributes)
        return defaults

    @variable_attributes.setter
    def variable_attributes(self, vas):
        self._variable_attributes = vas

    @property
    def global_attributes(self):
        gas = self._global_attributes
        gas.update({
            'geospatial_vertical_positive': self.vertical_positive,
            'date_created': datetime.utcnow().strftime("%Y-%m-%dT%H:%M:00Z"),
            'Conventions': 'CF-1.6',
            'Metadata_conventions': 'Unidata Dataset Discovery v1.0',
            'featureType': 'profile',
            'cdm_data_type': 'Profile'
        })

        if not self.df.empty:
            # Time
            starting = self.df['time'].min()
            ending = self.df['time'].max()
            duration = "P%sS" % str(int(round((ending - starting).total_seconds())))
            gas.update({
                'time_coverage_start': starting.strftime("%Y-%m-%dT%H:%M:00Z"),
                'time_coverage_end': ending.strftime("%Y-%m-%dT%H:%M:00Z"),
                'time_coverage_duration': duration,
            })
            diffs = self.df['time'].unique()[1:] - self.df['time'].unique()[:-1]
            uniqs, inverse = np.unique(diffs, return_inverse=True)
            if uniqs.size > 1:
                time_diffs = diffs[np.bincount(inverse).argmax()]
                gas.update({
                    'time_coverage_resolution': "P%sS" % str(round(time_diffs.astype('timedelta64[s]').astype(int)))
                })

            # Vertical
            gas.update({
                'geospatial_vertical_min': self.df['z'].min(),
                'geospatial_vertical_max': self.df['z'].max(),
            })

            # Horizontal
            gas.update({
                'geospatial_lat_min': self.df['latitude'].min(),
                'geospatial_lat_max': self.df['latitude'].max(),
                'geospatial_lon_min': self.df['longitude'].min(),
                'geospatial_lon_max': self.df['longitude'].max(),
            })

        return gas

    @global_attributes.setter
    def global_attributes(self, gas):
        # These are set by this script, we don't someone to be able to set them manually
        global_skips = ["time_coverage_start", "time_coverage_end", "time_coverage_duration", "time_coverage_resolution",
                        "featureType", "geospatial_vertical_positive", "geospatial_vertical_min", "geospatial_vertical_max",
                        "geospatial_lat_min", "geospatial_lon_min", "geospatial_lat_max", "geospatial_lon_max",
                        "Conventions", "date_created", "cdm_data_type"]

        for i in set(global_skips) & gas.keys():
            logger.warning("Ignoring global attribute {} because it is calculated or set automatically".format(i))

        self._global_attributes = { k: v for k, v in gas.items() if k not in global_skips }

    def export(self, output_file, file_type=None):
        # Make output directory
        if not os.path.exists(os.path.dirname(output_file)):
            os.makedirs(os.path.dirname(output_file))

class IncompleteProfile(Profile):

    def export(self, output_file):
        super(IncompleteProfile, self).export(output_file)

        with netCDF4.Dataset(output_file, 'w', clobber=True) as nc:

            gas = self.global_attributes
            nc.setncatts(gas)

            profiles = self.df.profile.unique().size
            profile_group = self.df.groupby('profile')
            max_z = profile_group.size().max()

            nc.createDimension('profile', profiles)
            nc.createDimension('z', max_z)

            profile = nc.createVariable('profile', self.df.profile.dtype, ('profile',))
            _, unique_profile_rows = np.unique(self.df.profile.values, return_index=True)
            profile[:] = list(range(profiles))

            time = nc.createVariable('time', int, ('profile',))
            time[:] = netCDF4.date2num([datetime.utcfromtimestamp(t) for t in self.df.time.unique().astype('<M8[s]').astype(int)], units=self.base_time)

            latitude = nc.createVariable('latitude', self.df.latitude.dtype, ('profile',))
            latitude[:] = self.df.latitude.values[unique_profile_rows]

            longitude = nc.createVariable('longitude', self.df.longitude.dtype, ('profile',))
            longitude[:] = self.df.longitude.values[unique_profile_rows]

            # Metadata variables
            nc.createVariable("crs", 'i4')
            nc.createVariable("platform", "i4")
            nc.setncattr('platform', 'platform')

            # Data vars
            reserved_columns = ['profile', 'time', 'latitude', 'longitude']
            for i, (name, p) in enumerate(profile_group):
                for c in [d for d in self.df.columns if d not in reserved_columns]:
                    var_name = c.split(' ')[0].lower()
                    fill = self.fill_value.astype(p[c].dtype)
                    if var_name not in nc.variables:
                        v = nc.createVariable(var_name, self.df[c].dtype, ('profile', 'z'), fill_value=fill)
                    else:
                        v = nc.variables[var_name]
                    assignable_values = p[c].fillna(fill).values
                    v[i, :len(assignable_values)] = assignable_values

            for k, v in self.variable_attributes.items():
                if k in nc.variables:
                    for n, z in v.items():
                        try:
                            nc.variables[k].setncattr(n, z)
                        except BaseException:
                            logger.warning('Could not set attribute {} on {}'.format(n, k))
<|endoftext|>


<commit_before><commit_msg>subcommands convert and analyse implemented in pmdsys.py<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
daemontools service module. This module will create daemontools type
service watcher.

This module is compatible with the :mod:`service <salt.states.service>` states,
so it can be used to maintain services using the ``provider`` argument:

.. code-block:: yaml

    myservice:
      service.running:
        - provider: daemontools
'''
from __future__ import absolute_import

# Import python libs
import logging
import os
import os.path
import re

# Import salt libs
from salt.exceptions import CommandExecutionError

# Function alias to not shadow built-ins.
__func_alias__ = {
    'reload_': 'reload'
}

log = logging.getLogger(__name__)

VALID_SERVICE_DIRS = [
    '/service',
    '/var/service',
    '/etc/service',
]
SERVICE_DIR = None
for service_dir in VALID_SERVICE_DIRS:
    if os.path.exists(service_dir):
        SERVICE_DIR = service_dir
        break

def _service_path(name):
    '''
    build service path
    '''
    if not SERVICE_DIR:
        raise CommandExecutionError("Could not find service directory.")
    return '{0}/{1}'.format(SERVICE_DIR, name)

#-- states.service  compatible args
def start(name):
    '''
    Starts service via daemontools

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.start <service name>
    '''
    __salt__['file.remove']('{0}/down'.format(_service_path(name)))
    cmd = 'svc -u {0}'.format(_service_path(name))
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

#-- states.service compatible args
def stop(name):
    '''
    Stops service via daemontools

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.stop <service name>
    '''
    __salt__['file.touch']('{0}/down'.format(_service_path(name)))
    cmd = 'svc -d {0}'.format(_service_path(name))
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def term(name):
    '''
    Send a TERM to service via daemontools

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.term <service name>
    '''
    cmd = 'svc -t {0}'.format(_service_path(name))
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

#-- states.service compatible
def reload_(name):
    '''
    Wrapper for term()

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.reload <service name>
    '''
    term(name)

#-- states.service compatible
def restart(name):
    '''
    Restart service via daemontools. This will stop/start service

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.restart <service name>
    '''
    ret = 'restart False'
    if stop(name) and start(name):
        ret = 'restart True'
    return ret

#-- states.service compatible
def full_restart(name):
    '''
    Calls daemontools.restart() function

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.full_restart <service name>
    '''
    restart(name)

#-- states.service compatible
def status(name, sig=None):
    '''
    Return the status for a service via daemontools, return pid if running

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.status <service name>
    '''
    cmd = 'svstat {0}'.format(_service_path(name))
    out = __salt__['cmd.run_stdout'](cmd, python_shell=False)
    try:
        pid = re.search(r'\(pid (\d+)\)', out).group(1)
    except AttributeError:
        pid = ''
    return pid

def available(name):
    '''
    Returns ``True`` if the specified service is available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.available foo
    '''
    return name in get_all()

def missing(name):
    '''
    The inverse of daemontools.available.
    Returns ``True`` if the specified service is not available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.missing foo
    '''
    return name not in get_all()

def get_all():
    '''
    Return a list of all available services

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.get_all
    '''
    if not SERVICE_DIR:
        raise CommandExecutionError("Could not find service directory.")
    #- List all daemontools services in
    return sorted(os.listdir(SERVICE_DIR))

def enabled(name, **kwargs):
    '''
    Return True if the named service is enabled, false otherwise
    A service is considered enabled if in your service directory:
    - an executable ./run file exist
    - a file named "down" does not exist

    name
        Service name

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.enabled <service name>
    '''
    if not available(name):
        log.error('Service {0} not found'.format(name))
        return False

    run_file = os.path.join(SERVICE_DIR, name, 'run')
    down_file = os.path.join(SERVICE_DIR, name, 'down')

    return (
        os.path.isfile(run_file)
        and os.access(run_file, os.X_OK)
        and not os.path.isfile(down_file)
    )

def disabled(name):
    '''
    Return True if the named service is enabled, false otherwise

    CLI Example:

    .. code-block:: bash

        salt '*' service.daemontools <service name>
    '''
    return not enabled(name)

<commit_msg>fix pep8 issues<commit_after># -*- coding: utf-8 -*-
'''
daemontools service module. This module will create daemontools type
service watcher.

This module is compatible with the :mod:`service <salt.states.service>` states,
so it can be used to maintain services using the ``provider`` argument:

.. code-block:: yaml

    myservice:
      service.running:
        - provider: daemontools
'''
from __future__ import absolute_import

# Import python libs
import logging
import os
import os.path
import re

# Import salt libs
from salt.exceptions import CommandExecutionError

# Function alias to not shadow built-ins.
__func_alias__ = {
    'reload_': 'reload'
}

log = logging.getLogger(__name__)

VALID_SERVICE_DIRS = [
    '/service',
    '/var/service',
    '/etc/service',
]
SERVICE_DIR = None
for service_dir in VALID_SERVICE_DIRS:
    if os.path.exists(service_dir):
        SERVICE_DIR = service_dir
        break

def _service_path(name):
    '''
    build service path
    '''
    if not SERVICE_DIR:
        raise CommandExecutionError("Could not find service directory.")
    return '{0}/{1}'.format(SERVICE_DIR, name)

#-- states.service  compatible args
def start(name):
    '''
    Starts service via daemontools

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.start <service name>
    '''
    __salt__['file.remove']('{0}/down'.format(_service_path(name)))
    cmd = 'svc -u {0}'.format(_service_path(name))
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

#-- states.service compatible args
def stop(name):
    '''
    Stops service via daemontools

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.stop <service name>
    '''
    __salt__['file.touch']('{0}/down'.format(_service_path(name)))
    cmd = 'svc -d {0}'.format(_service_path(name))
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def term(name):
    '''
    Send a TERM to service via daemontools

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.term <service name>
    '''
    cmd = 'svc -t {0}'.format(_service_path(name))
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

#-- states.service compatible
def reload_(name):
    '''
    Wrapper for term()

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.reload <service name>
    '''
    term(name)

#-- states.service compatible
def restart(name):
    '''
    Restart service via daemontools. This will stop/start service

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.restart <service name>
    '''
    ret = 'restart False'
    if stop(name) and start(name):
        ret = 'restart True'
    return ret

#-- states.service compatible
def full_restart(name):
    '''
    Calls daemontools.restart() function

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.full_restart <service name>
    '''
    restart(name)

#-- states.service compatible
def status(name, sig=None):
    '''
    Return the status for a service via daemontools, return pid if running

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.status <service name>
    '''
    cmd = 'svstat {0}'.format(_service_path(name))
    out = __salt__['cmd.run_stdout'](cmd, python_shell=False)
    try:
        pid = re.search(r'\(pid (\d+)\)', out).group(1)
    except AttributeError:
        pid = ''
    return pid

def available(name):
    '''
    Returns ``True`` if the specified service is available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.available foo
    '''
    return name in get_all()

def missing(name):
    '''
    The inverse of daemontools.available.
    Returns ``True`` if the specified service is not available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.missing foo
    '''
    return name not in get_all()

def get_all():
    '''
    Return a list of all available services

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.get_all
    '''
    if not SERVICE_DIR:
        raise CommandExecutionError("Could not find service directory.")
    #- List all daemontools services in
    return sorted(os.listdir(SERVICE_DIR))

def enabled(name, **kwargs):
    '''
    Return True if the named service is enabled, false otherwise
    A service is considered enabled if in your service directory:
    - an executable ./run file exist
    - a file named "down" does not exist

    name
        Service name

    CLI Example:

    .. code-block:: bash

        salt '*' daemontools.enabled <service name>
    '''
    if not available(name):
        log.error('Service {0} not found'.format(name))
        return False

    run_file = os.path.join(SERVICE_DIR, name, 'run')
    down_file = os.path.join(SERVICE_DIR, name, 'down')

    return (
        os.path.isfile(run_file) and
        os.access(run_file, os.X_OK) and not
        os.path.isfile(down_file)
    )

def disabled(name):
    '''
    Return True if the named service is enabled, false otherwise

    CLI Example:

    .. code-block:: bash

        salt '*' service.daemontools <service name>
    '''
    return not enabled(name)

<|endoftext|>