<commit_before>from os.path import join, exists, isdir, relpath, abspath, dirname
import datetime as dt
import posixpath
import logging
import tempfile
from os import stat, makedirs, remove
import random
import uuid
import cStringIO
import time

from six import string_types
try:
    import gevent
except:
    gevent = None
from ..clients.http import Client
from .. import settings
from ..serialization import deserializer, serializer
from ..errors import KitchenSinkError
from ..utils.pathutils import urlsplit, dirsplit, urljoin

logger = logging.getLogger(__name__)

def hosts():
    if settings.prefix:
        host_key = settings.prefix + ":" + "hosts"
        hostinfo_key = settings.prefix + ":" + "hostinfo"
    else:
        host_key = "hosts"
        hostinfo_key = "hostinfo"
    possible_hosts = settings.redis_conn.smembers(host_key)
    hosts = settings.redis_conn.mget(*possible_hosts)
    hosts = [x for x in hosts if x is not None]
    return hosts

class Catalog(object):
    def __init__(self, connection, datadir, host_url, prefix=None):
        """connection - redis connection
        host_url - url of current controller/host
        """
        if prefix is None:
            prefix = settings.prefix
        self.prefix = ""
        self.conn = connection
        self.datadir = datadir
        self.host_url = host_url

    def path_key(self, key):
        if self.prefix:
            return self.prefix + ":data:path:" + key
        else:
            return "data:path:" + key

    def starting_key(self, key):
        if self.prefix:
            return self.prefix + ":datastarting:path:" + key
        else:
            return "datastarting:path:" + key

    def data_key(self, key):
        if self.prefix:
            return self.prefix + ":data:info:" + key
        else:
            return "data:info:" + key

    def search(self, pattern):
        if self.prefix:
            prefix = self.prefix + ":data:info:"
        else:
            prefix = self.prefix + "data:info:"
        pattern = prefix + pattern
        ### implement this with scan later
        keys = self.conn.keys(pattern)
        return [x[len(prefix):] for x in keys]

    def _raw_write(self, finput, f):
        if not isinstance(finput, string_types):
            while True:
                data = finput.read(settings.chunk_size)
                if data:
                    f.write(data)
                else:
                    break
        else:
            f.write(finput)
    def setup_file_path_from_url(self, url):
        splits = urlsplit(url, "")
        file_path = abspath(join(self.datadir, *splits))
        if not file_path.startswith(self.datadir):
            raise KitchenSinkError, "Must be inside datadir"
        if not exists(dirname(file_path)):
            makedirs(dirname(file_path))
        return file_path

    def write(self, finput, url, is_new=True, data_type="object", fmt="cloudpickle"):
        file_path = self.setup_file_path_from_url(url)
        if is_new:
            if self.url_exists(url):
                raise KitchenSinkError("path already being used")
            else:
                self.init_addition(file_path, url)
        if not exists(file_path):
            print ("WRITING", file_path)
            with open(file_path, "wb+") as f:
                self._raw_write(finput, f)
        if is_new:
            size = stat(file_path).st_size
            self.set_metadata(url, size, data_type=data_type, fmt=fmt)
        self.add(file_path, url)
        return file_path

    def delete(self, url):
        start_key = self.starting_key(url)
        path_key = self.path_key(url)
        data_key = self.data_key(url)
        self.conn.hdel(path_key, self.host_url)
        self.conn.hdel(start_key, self.host_url)
        if not self.conn.exists(path_key):
            self.conn.delete(data_key)
        file_path = self.setup_file_path_from_url(url)
        remove(file_path)

    def write_chunked(self, iterator, url, is_new=True):
        file_path = self.setup_file_path_from_url(url)
        if is_new:
            if self.url_exists(url):
                raise KitchenSinkError("path already being used")
            else:
                self.init_addition(file_path, url)
        with open(file_path, "wb+") as f:
            for chunk in iterator:
                f.write(chunk)
        if is_new:
            size = stat(file_path).st_size
            self.set_metadata(url, size, data_type=data_type)
        self.add(file_path, url)
        return file_path

    def url_exists(self, url):
        start_key = self.starting_key(url)
        path_key = self.path_key(url)
        data_key = self.data_key(url)
        return self.conn.exists(start_key) or \
            self.conn.exists(path_key) or \
            self.conn.exists(data_key)

    def add(self, file_path, url):
        path_key = self.path_key(url)
        self.conn.hset(path_key, self.host_url, file_path)
        return file_path

    def init_addition(self, file_path, url):
        start_key = self.starting_key(url)
        self.conn.hset(start_key, self.host_url, file_path)

    def set_metadata(self, url, size, data_type="object", fmt="cloudpickle"):
        data_key = self.data_key(url)
        self.conn.hmset(data_key, {'size' : str(size),
                                   'data_type' : data_type,
                                   'fmt' : fmt}
        )

    def get_chunked_iterator(self, url, length, host_url=None):
        if host_url is None:
            host_url = self.host_url
        data_read = 0
        c = Client(host_url, rpc_name='data', queue_name='data')
        while True:
            if data_read == length:
                break
            #logger.info("%s, %s, %s", url, data_read, settings.chunk_size)
            data = c._get_data(url, data_read, settings.chunk_size)
            if data.status_code != 200:
                raise KitchenSinkError("http error")
            data = data.raw.read()
            data_read += len(data)
            logger.info ("read %s of %s from %s to %s" % (data_read, length,
                                                          host_url, settings.host_url))
            if gevent:
                gevent.sleep(0)
            if len(data) == 0:
                time.sleep(1.0)
            yield data

    def get(self, url, host=None):
        file_path = self.setup_file_path_from_url(url)
        if exists(file_path):
            logger.warning("ALREADY HAVE %s, no need to retrieve" % file_path)
            return file_path
        hosts_info, data_info = self.get_info(url)
        if self.host_url in hosts_info:
            return hosts_info[self.host_url]
        else:
            host = hosts_info.keys()[0]
            c = Client(host, rpc_name='data', queue_name='data')
            stream = None
            try:
                stream = c._get_data(url).raw
                retval = self.write(stream, url, is_new=False)
            finally:
                if stream:
                    stream.close()
            return retval

    def get_info(self, url):
        hosts_info = self.conn.hgetall(self.path_key(url))
        data_info = self.conn.hgetall(self.data_key(url))
        if 'size' in data_info:
            data_info['size'] = int(data_info['size'])
        return (hosts_info, data_info)

    def get_file_path(self, url, unfinished=False):
        if unfinished:
            return self.setup_file_path_from_url(url)
        hosts_info, data_info = self.get_info(url)
        file_path = hosts_info[self.host_url]
        if not file_path.startswith(self.datadir):
            raise KitchenSinkError, "Must be inside datadir"
        return file_path

## should factor this out into a separate module
class RemoteData(object):
    def __init__(self, obj=None, local_path=None, data_url=None, rpc_url=None,
                 fmt="cloudpickle"):
        if rpc_url is None:
            rpc_url = settings.data_rpc_url
        self.rpc_url = rpc_url
        self.data_url = data_url
        self.fmt = fmt
        self._obj = obj
        self._local_path = local_path
        self._raw = None

    def __setstate__(self, obj):
        self.data_url = obj['data_url']
        self.fmt = obj['fmt']
        self.rpc_url = settings.data_rpc_url
        self._obj = None
        self._local_path = None
        self._raw = None

    def __getstate__(self):
        return dict(data_url=self.data_url,
                    fmt=self.fmt)

    def client(self, rpc_url=None):
        if rpc_url is None:
            rpc_url = self.rpc_url
        return Client(rpc_url, rpc_name='data', queue_name='data')

    def _get(self):
        if settings.catalog:
            return settings.catalog.get(self.data_url)
        c = self.client(self.rpc_url)
        url = c.pick_host(self.data_url)
        resp = self.client(url)._get_data(self.data_url)
        return resp.raw

    def _put(self, f, data_type='object', fmt="cloudpickle"):
        logger.info("posting %s to %s", self.data_url, self.rpc_url)
        f.seek(0)
        c = self.client(self.rpc_url)
        return c._put_data(self.data_url, f, data_type=data_type, fmt=fmt)

    def local_path(self, path=None):
        """provides a path to a local file that contains the
        contents of this remote data source (downloads if necessary)
        """
        if self._local_path:
            return self._local_path
        name = tempfile.NamedTemporaryFile(prefix="ks-data-").name
        if self._raw:
            with open(name, "w+") as f:
                f.write(self._raw)
        else:
            stream = self._get()
            #not great, but this means we're getting back the file path
            if isinstance(stream, string_types):
                name = stream
                return name
            try:
                name = tempfile.NamedTemporaryFile(prefix="ks-data-").name
                with open(name, "w+") as f:
                    while True:
                        data = stream.read(settings.chunk_size)
                        if data:
                            f.write(data)
                        else:
                            break
            finally:
                stream.close()
        self._local_path = name
        return name

    def raw(self):
        """provides raw contents of the remote data
        """
        if self._raw:
            return self._raw
        raw = None
        if self._local_path:
            with open(self._local_path) as f:
                raw = f.read()
        else:
            raw = self._get()
            if isinstance(raw, string_types):
                name = raw
                with open(name, "r") as f:
                    raw = f.read()
            else:
                try:
                    stream = raw
                    raw = stream.read()
                finally:
                    stream.close()
        self._raw = raw
        return raw

    def obj(self):
        if self._obj is not None:
            return self._obj
        else:
            #should be able to pass a file in later
            try:
                raw = self.raw()
                obj = deserializer(self.fmt)(raw)
                self._obj = obj
                return obj
            except Exception as e:
                logger.error("error with %s on %s",
                             self.data_url,
                             settings.data_rpc_url)
                logger.exception(e)
                raise
    def delete(self):
        raise NotImplementedError

    def save(self, url=None, prefix=""):
        """use this function to save a NEW data object
        """
        if self.data_url:
            raise KitchenSinkError, "Dataset is already created, cannot be modified"
        if url is None:
            self.data_url = urljoin(prefix, str(uuid.uuid4()))
        else:
            self.data_url = url
        try:
            f = None
            if self._raw:
                f = cStringIO.StringIO()
                f.write(self._raw)
            elif self._local_path:
                f = open(self._local_path, "rb")
            else:
                f = cStringIO.StringIO()
                data = serializer(self.fmt)(self._obj)
                f.write(data)
            data_type = "object" if self._obj is not None else None
            fmt = self.fmt if self._obj is not None else None
            self._put(f, data_type=data_type, fmt=fmt)
        except Exception as e:
            if f:
                f.close()
            raise e

    def pipeline(self, existing=False, url=None, prefix=""):
        if existing:
            return self.pipeline_existing()
        if self.data_url and not existing:
            raise KitchenSinkError, "Dataset is already created, cannot be modified"
        if not self.data_url:
            if url is None:
                self.data_url = urljoin(prefix,
                                        str(uuid.uuid4()))
            else:
                self.data_url = url
        try:
            if self._raw:
                f = cStringIO.StringIO()
                f.write(self._raw)
                length = len(self._raw)
            elif self._local_path:
                f = open(self._local_path)
                length = stat(file_path).self._local_path
            else:
                f = cStringIO.StringIO()
                data = serializer(self.fmt)(self._obj)
                f.write(data)
                length = len(data)
            c = self.client()
            host = self.rpc_url
            calls = self._pipeline_existing(host, length=length)
            self._put(f)
            print c.bulk_async_result(calls)
        except Exception as e:
            f.close()
            raise e

    def _pipeline_existing(self, starting_host, length=None):
        c = self.client()
        host_info, data_info = c.call('get_info', self.data_url,
                                      _rpc_name='data',
                                      _async=False)
        if length is None:
            length = data_info['size']
        hosts = c.call('hosts', _async=False)
        hosts = [x for x in hosts if x not in host_info]
        hosts = [x for x in hosts if x != starting_host]
        hosts.append(starting_host)
        print hosts
        calls = []
        for idx in range(1, len(hosts)):
            result = c.call('chunked_copy', self.data_url, length, hosts[idx],
                            _queue_name=hosts[idx - 1],
            )
            calls.append(result)
        return calls

    def pipeline_existing(self):
        c = self.client()
        host = c.pick_host(self.data_url)
        calls = self._pipeline_existing(host)
        print c.bulk_async_result(calls)
    def __repr__(self):
        if self.data_url:
            return "RemoteData(data_url='%s')" % self.data_url
        elif self._local_path:
            return "RemoteData(local_path='%s')" % self._local_path
        else:
            return "RemoteData(obj=obj)"

    def __getitem__(self, arg):
        if settings.catalog:
            print ("loading direct")
            obj = do(self.obj()[arg])
            obj.save()
            return obj
        else:
            c = Client(self.rpc_url)
            return c.async_result(c.call(self.__class__.__getitem__, self, arg))

    __getitem__.ks_memoize = True

def du(url):
    return RemoteData(data_url=url)
def dp(path):
    return RemoteData(local_path=path)
def do(obj):
    return RemoteData(obj=obj)
<commit_msg>more logging<commit_after>from os.path import join, exists, isdir, relpath, abspath, dirname
import datetime as dt
import posixpath
import logging
import tempfile
from os import stat, makedirs, remove
import random
import uuid
import cStringIO
import time

from six import string_types
try:
    import gevent
except:
    gevent = None
from ..clients.http import Client
from .. import settings
from ..serialization import deserializer, serializer
from ..errors import KitchenSinkError
from ..utils.pathutils import urlsplit, dirsplit, urljoin

logger = logging.getLogger(__name__)

def hosts():
    if settings.prefix:
        host_key = settings.prefix + ":" + "hosts"
        hostinfo_key = settings.prefix + ":" + "hostinfo"
    else:
        host_key = "hosts"
        hostinfo_key = "hostinfo"
    possible_hosts = settings.redis_conn.smembers(host_key)
    hosts = settings.redis_conn.mget(*possible_hosts)
    hosts = [x for x in hosts if x is not None]
    return hosts

class Catalog(object):
    def __init__(self, connection, datadir, host_url, prefix=None):
        """connection - redis connection
        host_url - url of current controller/host
        """
        if prefix is None:
            prefix = settings.prefix
        self.prefix = ""
        self.conn = connection
        self.datadir = datadir
        self.host_url = host_url

    def path_key(self, key):
        if self.prefix:
            return self.prefix + ":data:path:" + key
        else:
            return "data:path:" + key

    def starting_key(self, key):
        if self.prefix:
            return self.prefix + ":datastarting:path:" + key
        else:
            return "datastarting:path:" + key

    def data_key(self, key):
        if self.prefix:
            return self.prefix + ":data:info:" + key
        else:
            return "data:info:" + key

    def search(self, pattern):
        if self.prefix:
            prefix = self.prefix + ":data:info:"
        else:
            prefix = self.prefix + "data:info:"
        pattern = prefix + pattern
        ### implement this with scan later
        keys = self.conn.keys(pattern)
        return [x[len(prefix):] for x in keys]

    def _raw_write(self, finput, f):
        if not isinstance(finput, string_types):
            while True:
                data = finput.read(settings.chunk_size)
                if data:
                    f.write(data)
                else:
                    break
        else:
            f.write(finput)
    def setup_file_path_from_url(self, url):
        splits = urlsplit(url, "")
        file_path = abspath(join(self.datadir, *splits))
        if not file_path.startswith(self.datadir):
            raise KitchenSinkError, "Must be inside datadir"
        if not exists(dirname(file_path)):
            makedirs(dirname(file_path))
        return file_path

    def write(self, finput, url, is_new=True, data_type="object", fmt="cloudpickle"):
        file_path = self.setup_file_path_from_url(url)
        if is_new:
            if self.url_exists(url):
                raise KitchenSinkError("path already being used")
            else:
                self.init_addition(file_path, url)
        if not exists(file_path):
            print ("WRITING", file_path)
            with open(file_path, "wb+") as f:
                self._raw_write(finput, f)
        if is_new:
            size = stat(file_path).st_size
            self.set_metadata(url, size, data_type=data_type, fmt=fmt)
        self.add(file_path, url)
        return file_path

    def delete(self, url):
        start_key = self.starting_key(url)
        path_key = self.path_key(url)
        data_key = self.data_key(url)
        self.conn.hdel(path_key, self.host_url)
        self.conn.hdel(start_key, self.host_url)
        if not self.conn.exists(path_key):
            self.conn.delete(data_key)
        file_path = self.setup_file_path_from_url(url)
        remove(file_path)

    def write_chunked(self, iterator, url, is_new=True):
        file_path = self.setup_file_path_from_url(url)
        if is_new:
            if self.url_exists(url):
                raise KitchenSinkError("path already being used")
            else:
                self.init_addition(file_path, url)
        with open(file_path, "wb+") as f:
            for chunk in iterator:
                f.write(chunk)
        if is_new:
            size = stat(file_path).st_size
            self.set_metadata(url, size, data_type=data_type)
        self.add(file_path, url)
        return file_path

    def url_exists(self, url):
        start_key = self.starting_key(url)
        path_key = self.path_key(url)
        data_key = self.data_key(url)
        return self.conn.exists(start_key) or \
            self.conn.exists(path_key) or \
            self.conn.exists(data_key)

    def add(self, file_path, url):
        path_key = self.path_key(url)
        self.conn.hset(path_key, self.host_url, file_path)
        return file_path

    def init_addition(self, file_path, url):
        start_key = self.starting_key(url)
        self.conn.hset(start_key, self.host_url, file_path)

    def set_metadata(self, url, size, data_type="object", fmt="cloudpickle"):
        data_key = self.data_key(url)
        self.conn.hmset(data_key, {'size' : str(size),
                                   'data_type' : data_type,
                                   'fmt' : fmt}
        )

    def get_chunked_iterator(self, url, length, host_url=None):
        if host_url is None:
            host_url = self.host_url
        data_read = 0
        c = Client(host_url, rpc_name='data', queue_name='data')
        while True:
            if data_read == length:
                break
            #logger.info("%s, %s, %s", url, data_read, settings.chunk_size)
            data = c._get_data(url, data_read, settings.chunk_size)
            if data.status_code != 200:
                raise KitchenSinkError("http error")
            data = data.raw.read()
            data_read += len(data)
            logger.info ("read %s of %s from %s to %s" % (data_read, length,
                                                          host_url, settings.host_url))
            if gevent:
                gevent.sleep(0)
            if len(data) == 0:
                time.sleep(1.0)
            yield data

    def get(self, url, host=None):
        file_path = self.setup_file_path_from_url(url)
        if exists(file_path):
            logger.warning("ALREADY HAVE %s, no need to retrieve" % file_path)
            return file_path
        hosts_info, data_info = self.get_info(url)
        if self.host_url in hosts_info:
            return hosts_info[self.host_url]
        else:
            host = hosts_info.keys()[0]
            c = Client(host, rpc_name='data', queue_name='data')
            stream = None
            try:
                stream = c._get_data(url).raw
                retval = self.write(stream, url, is_new=False)
            finally:
                if stream:
                    stream.close()
            return retval

    def get_info(self, url):
        hosts_info = self.conn.hgetall(self.path_key(url))
        data_info = self.conn.hgetall(self.data_key(url))
        if 'size' in data_info:
            data_info['size'] = int(data_info['size'])
        return (hosts_info, data_info)

    def get_file_path(self, url, unfinished=False):
        if unfinished:
            return self.setup_file_path_from_url(url)
        hosts_info, data_info = self.get_info(url)
        file_path = hosts_info[self.host_url]
        if not file_path.startswith(self.datadir):
            raise KitchenSinkError, "Must be inside datadir"
        return file_path

## should factor this out into a separate module
class RemoteData(object):
    def __init__(self, obj=None, local_path=None, data_url=None, rpc_url=None,
                 fmt="cloudpickle"):
        if rpc_url is None:
            rpc_url = settings.data_rpc_url
        self.rpc_url = rpc_url
        self.data_url = data_url
        self.fmt = fmt
        self._obj = obj
        self._local_path = local_path
        self._raw = None

    def __setstate__(self, obj):
        self.data_url = obj['data_url']
        self.fmt = obj['fmt']
        self.rpc_url = settings.data_rpc_url
        self._obj = None
        self._local_path = None
        self._raw = None

    def __getstate__(self):
        return dict(data_url=self.data_url,
                    fmt=self.fmt)

    def client(self, rpc_url=None):
        if rpc_url is None:
            rpc_url = self.rpc_url
        return Client(rpc_url, rpc_name='data', queue_name='data')

    def _get(self):
        if settings.catalog:
            return settings.catalog.get(self.data_url)
        c = self.client(self.rpc_url)
        url = c.pick_host(self.data_url)
        resp = self.client(url)._get_data(self.data_url)
        return resp.raw

    def _put(self, f, data_type='object', fmt="cloudpickle"):
        logger.info("posting %s to %s", self.data_url, self.rpc_url)
        f.seek(0)
        c = self.client(self.rpc_url)
        return c._put_data(self.data_url, f, data_type=data_type, fmt=fmt)

    def local_path(self, path=None):
        """provides a path to a local file that contains the
        contents of this remote data source (downloads if necessary)
        """
        if self._local_path:
            return self._local_path
        name = tempfile.NamedTemporaryFile(prefix="ks-data-").name
        if self._raw:
            with open(name, "w+") as f:
                f.write(self._raw)
        else:
            stream = self._get()
            #not great, but this means we're getting back the file path
            if isinstance(stream, string_types):
                name = stream
                return name
            try:
                name = tempfile.NamedTemporaryFile(prefix="ks-data-").name
                with open(name, "w+") as f:
                    while True:
                        data = stream.read(settings.chunk_size)
                        if data:
                            f.write(data)
                        else:
                            break
            finally:
                stream.close()
        self._local_path = name
        return name

    def raw(self):
        """provides raw contents of the remote data
        """
        if self._raw:
            return self._raw
        raw = None
        if self._local_path:
            with open(self._local_path) as f:
                raw = f.read()
        else:
            raw = self._get()
            if isinstance(raw, string_types):
                name = raw
                with open(name, "r") as f:
                    raw = f.read()
            else:
                try:
                    stream = raw
                    raw = stream.read()
                finally:
                    stream.close()
        self._raw = raw
        return raw

    def obj(self):
        if self._obj is not None:
            return self._obj
        else:
            #should be able to pass a file in later
            try:
                raw = self.raw()
                obj = deserializer(self.fmt)(raw)
                self._obj = obj
                return obj
            except Exception as e:
                logger.error("error with %s on %s raw len %s",
                             self.data_url,
                             settings.data_rpc_url,
                             len(raw)
                )
                logger.exception(e)
                raise
    def delete(self):
        raise NotImplementedError

    def save(self, url=None, prefix=""):
        """use this function to save a NEW data object
        """
        if self.data_url:
            raise KitchenSinkError, "Dataset is already created, cannot be modified"
        if url is None:
            self.data_url = urljoin(prefix, str(uuid.uuid4()))
        else:
            self.data_url = url
        try:
            f = None
            if self._raw:
                f = cStringIO.StringIO()
                f.write(self._raw)
            elif self._local_path:
                f = open(self._local_path, "rb")
            else:
                f = cStringIO.StringIO()
                data = serializer(self.fmt)(self._obj)
                f.write(data)
            data_type = "object" if self._obj is not None else None
            fmt = self.fmt if self._obj is not None else None
            self._put(f, data_type=data_type, fmt=fmt)
        except Exception as e:
            if f:
                f.close()
            raise e

    def pipeline(self, existing=False, url=None, prefix=""):
        if existing:
            return self.pipeline_existing()
        if self.data_url and not existing:
            raise KitchenSinkError, "Dataset is already created, cannot be modified"
        if not self.data_url:
            if url is None:
                self.data_url = urljoin(prefix,
                                        str(uuid.uuid4()))
            else:
                self.data_url = url
        try:
            if self._raw:
                f = cStringIO.StringIO()
                f.write(self._raw)
                length = len(self._raw)
            elif self._local_path:
                f = open(self._local_path)
                length = stat(file_path).self._local_path
            else:
                f = cStringIO.StringIO()
                data = serializer(self.fmt)(self._obj)
                f.write(data)
                length = len(data)
            c = self.client()
            host = self.rpc_url
            calls = self._pipeline_existing(host, length=length)
            self._put(f)
            print c.bulk_async_result(calls)
        except Exception as e:
            f.close()
            raise e

    def _pipeline_existing(self, starting_host, length=None):
        c = self.client()
        host_info, data_info = c.call('get_info', self.data_url,
                                      _rpc_name='data',
                                      _async=False)
        if length is None:
            length = data_info['size']
        hosts = c.call('hosts', _async=False)
        hosts = [x for x in hosts if x not in host_info]
        hosts = [x for x in hosts if x != starting_host]
        hosts.append(starting_host)
        print hosts
        calls = []
        for idx in range(1, len(hosts)):
            result = c.call('chunked_copy', self.data_url, length, hosts[idx],
                            _queue_name=hosts[idx - 1],
            )
            calls.append(result)
        return calls

    def pipeline_existing(self):
        c = self.client()
        host = c.pick_host(self.data_url)
        calls = self._pipeline_existing(host)
        print c.bulk_async_result(calls)
    def __repr__(self):
        if self.data_url:
            return "RemoteData(data_url='%s')" % self.data_url
        elif self._local_path:
            return "RemoteData(local_path='%s')" % self._local_path
        else:
            return "RemoteData(obj=obj)"

    def __getitem__(self, arg):
        if settings.catalog:
            print ("loading direct")
            obj = do(self.obj()[arg])
            obj.save()
            return obj
        else:
            c = Client(self.rpc_url)
            return c.async_result(c.call(self.__class__.__getitem__, self, arg))

    __getitem__.ks_memoize = True

def du(url):
    return RemoteData(data_url=url)
def dp(path):
    return RemoteData(local_path=path)
def do(obj):
    return RemoteData(obj=obj)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2015 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import os
import stat
import json
import re
import six

from girder_client import GirderClient

import cumulus
from cumulus.transport import get_connection
from cumulus.transport.files import get_assetstore_url_base, get_assetstore_id

def _include(path, includes, excludes):
    """
    :params path: The file path to check whether to include
    :params includes: List of include regexs to apply
    :params excludes: List of exclude regexs to apply

    :returns True if the file path should be included, False

    """
    if includes is None:
        includes = []

    if excludes is None:
        excludes = []

    # If there are no regexs include the path
    if not includes and not excludes:
        return True

    include = not includes
    for i in includes:
        if re.match(i, path):
            include = True
            break

    for e in excludes:
        if re.match(e, path):
            include = False
            break

    return include

def _ensure_path(girder_client, girder_folders, parent, path):
    """
    Ensure that a particular path exists in Girder as a set of folders. If
    it doesn't exist create it.

    :params girder_client: The Girder client to use to access Girder.
    :params girder_folders: A map of paths to existing Girder folder ids.
    :params parent: The parent Girder folder that this path should exist in.
    :params path: The path that should exist.

    :returns The folderId for the last folder in the path.
    """
    if path in girder_folders:
        return girder_folders[path]

    parts = path.split('/')

    # Default our parent_id to parent, however this may be updated if we already
    # have folders created in Girder, see loop below.
    parent_id = parent

    # First identify the folders we have already have in Girder
    index = 0
    for i in range(len(parts), 0, -1):
        part = '/'.join(parts[:i])
        if part in girder_folders:
            parent_id = girder_folders[part]
            index = i
            break

    # Now walk through the parts of the path we need to create
    i = index + 1
    parent_created = False
    for name in parts[index:]:
        path = '/'.join(parts[:i])
        # Check if the folder already exists
        if not parent_created:
            folders = girder_client.listFolder(parent_id, name=name)
            try:
                folder_id = six.next(folders)['_id']
                girder_folders[path] = folder_id
                parent_id = folder_id
                i += 1
                continue
            except StopIteration:
                pass

        # Create the folder
        folder = girder_client.createFolder(parent_id, name,
                                            parentType='folder')
        # Don't bother checking if folders have already been created ...
        parent_created = True
        parent_id = folder['_id']
        girder_folders[path] = parent_id

        i += 1

    return parent_id

def _import_path(cluster_connection, girder_client, parent, root_path,
                 assetstore_url, assetstore_id, upload=False,
                 include=None, exclude=None, path='.', girder_folders=None):
    """
    :params cluster_connection: The cluster connection to access the cluster.
    :params girder_client: The Girder client to use to access Girder.
    :params parent: The target folder to import the path into.
    :params root_path: The path on the cluster being imported.
    :params assetstore_url: The url for the assetstore to use for the import.
    :params assetstore_id: The id of the asseststore to import into.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    :params path: The current subdirectory of root_path that is being imported.
    """
    if girder_folders is None:
        girder_folders = {}

    if root_path[0] != '/':
        # If we don't have a full path, assume the path is relative to the users
        # home directory.
        home = cluster_connection.execute('pwd')[0]
        root_path = os.path.abspath(os.path.join(home, root_path))

    cluster_path = os.path.normpath(os.path.join(root_path, path))
    for p in cluster_connection.list(cluster_path):
        name = p['name']

        full_path = os.path.normpath(os.path.join(path, name))
        if name in ['.', '..']:
            continue

        if stat.S_ISDIR(p['mode']):
            _import_path(cluster_connection, girder_client, parent,
                         root_path, assetstore_url, assetstore_id,
                         upload=upload, path=full_path,
                         include=include,
                         exclude=exclude, girder_folders=girder_folders)
        else:

            # Should we include this path?
            if not _include(full_path, include, exclude):
                continue

            # Create any folders we might need
            if path == '.':
                folder_id = parent
            else:
                folder_id = _ensure_path(girder_client, girder_folders, parent,
                                         path)

            size = p['size']
            item = girder_client.createItem(folder_id, name, '')

            if not upload:

                url = '%s/%s/files' % (assetstore_url, assetstore_id)
                body = {
                    'name': name,
                    'itemId': item['_id'],
                    'size': size,
                    'path': os.path.join(root_path, full_path)
                }
                girder_client.post(url, data=json.dumps(body))
            else:
                cluster_path = os.path.normpath(
                    os.path.join(root_path, full_path))
                with cluster_connection.get(cluster_path) as stream:
                    girder_client.uploadFile(item['_id'], stream, name, size,
                                             parentType='item')

def download_path(cluster_connection, girder_token, parent, path,
                  assetstore_url, assetstore_id, upload=False, include=None,
                  exclude=None):
    """
    Download a given path on a cluster into an assetstore.

    :params cluster_connection: The cluster connection to access the cluster.
    :params girder_token: The Girder token to use to access Girder.
    :params parent: The target folder to import the path into.
    :params path: The path on the cluster to download.
    :params assetstore_url: The url for the assetstore to use for the import.
    :params assetstore_id: The id of the asseststore to import into.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    """
    girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
    girder_client.token = girder_token

    _import_path(cluster_connection, girder_client, parent, path,
                 assetstore_url, assetstore_id, upload=upload, include=include,
                 exclude=exclude)

def download_path_from_cluster(cluster, girder_token, parent, path,
                               upload=False, include=None, exclude=None):
    """
    Download a given path on a cluster into an assetstore.

    :params cluster: The cluster to to download the path from.
    :params girder_token: The Girder token to use to access Girder.
    :params parent: The target folder to import the path into.
    :params path: The path on the cluster to download.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    """
    assetstore_base_url = get_assetstore_url_base(cluster)
    assetstore_id = get_assetstore_id(girder_token, cluster)

    with get_connection(girder_token, cluster) as conn:
        download_path(conn, girder_token, parent, path, assetstore_base_url,
                      assetstore_id, upload=upload, include=include,
                      exclude=exclude)
<commit_msg>Cast from unicode to integer<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2015 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import os
import stat
import json
import re
import six

from girder_client import GirderClient

import cumulus
from cumulus.transport import get_connection
from cumulus.transport.files import get_assetstore_url_base, get_assetstore_id

def _include(path, includes, excludes):
    """
    :params path: The file path to check whether to include
    :params includes: List of include regexs to apply
    :params excludes: List of exclude regexs to apply

    :returns True if the file path should be included, False

    """
    if includes is None:
        includes = []

    if excludes is None:
        excludes = []

    # If there are no regexs include the path
    if not includes and not excludes:
        return True

    include = not includes
    for i in includes:
        if re.match(i, path):
            include = True
            break

    for e in excludes:
        if re.match(e, path):
            include = False
            break

    return include

def _ensure_path(girder_client, girder_folders, parent, path):
    """
    Ensure that a particular path exists in Girder as a set of folders. If
    it doesn't exist create it.

    :params girder_client: The Girder client to use to access Girder.
    :params girder_folders: A map of paths to existing Girder folder ids.
    :params parent: The parent Girder folder that this path should exist in.
    :params path: The path that should exist.

    :returns The folderId for the last folder in the path.
    """
    if path in girder_folders:
        return girder_folders[path]

    parts = path.split('/')

    # Default our parent_id to parent, however this may be updated if we already
    # have folders created in Girder, see loop below.
    parent_id = parent

    # First identify the folders we have already have in Girder
    index = 0
    for i in range(len(parts), 0, -1):
        part = '/'.join(parts[:i])
        if part in girder_folders:
            parent_id = girder_folders[part]
            index = i
            break

    # Now walk through the parts of the path we need to create
    i = index + 1
    parent_created = False
    for name in parts[index:]:
        path = '/'.join(parts[:i])
        # Check if the folder already exists
        if not parent_created:
            folders = girder_client.listFolder(parent_id, name=name)
            try:
                folder_id = six.next(folders)['_id']
                girder_folders[path] = folder_id
                parent_id = folder_id
                i += 1
                continue
            except StopIteration:
                pass

        # Create the folder
        folder = girder_client.createFolder(parent_id, name,
                                            parentType='folder')
        # Don't bother checking if folders have already been created ...
        parent_created = True
        parent_id = folder['_id']
        girder_folders[path] = parent_id

        i += 1

    return parent_id

def _import_path(cluster_connection, girder_client, parent, root_path,
                 assetstore_url, assetstore_id, upload=False,
                 include=None, exclude=None, path='.', girder_folders=None):
    """
    :params cluster_connection: The cluster connection to access the cluster.
    :params girder_client: The Girder client to use to access Girder.
    :params parent: The target folder to import the path into.
    :params root_path: The path on the cluster being imported.
    :params assetstore_url: The url for the assetstore to use for the import.
    :params assetstore_id: The id of the asseststore to import into.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    :params path: The current subdirectory of root_path that is being imported.
    """
    if girder_folders is None:
        girder_folders = {}

    if root_path[0] != '/':
        # If we don't have a full path, assume the path is relative to the users
        # home directory.
        home = cluster_connection.execute('pwd')[0]
        root_path = os.path.abspath(os.path.join(home, root_path))

    cluster_path = os.path.normpath(os.path.join(root_path, path))
    for p in cluster_connection.list(cluster_path):
        name = p['name']

        full_path = os.path.normpath(os.path.join(path, name))
        if name in ['.', '..']:
            continue

        if stat.S_ISDIR(p['mode']):
            _import_path(cluster_connection, girder_client, parent,
                         root_path, assetstore_url, assetstore_id,
                         upload=upload, path=full_path,
                         include=include,
                         exclude=exclude, girder_folders=girder_folders)
        else:

            # Should we include this path?
            if not _include(full_path, include, exclude):
                continue

            # Create any folders we might need
            if path == '.':
                folder_id = parent
            else:
                folder_id = _ensure_path(girder_client, girder_folders, parent,
                                         path)

            size = int(p['size'])
            item = girder_client.createItem(folder_id, name, '')

            if not upload:

                url = '%s/%s/files' % (assetstore_url, assetstore_id)
                body = {
                    'name': name,
                    'itemId': item['_id'],
                    'size': size,
                    'path': os.path.join(root_path, full_path)
                }
                girder_client.post(url, data=json.dumps(body))
            else:
                cluster_path = os.path.normpath(
                    os.path.join(root_path, full_path))
                with cluster_connection.get(cluster_path) as stream:
                    girder_client.uploadFile(item['_id'], stream, name, size,
                                             parentType='item')

def download_path(cluster_connection, girder_token, parent, path,
                  assetstore_url, assetstore_id, upload=False, include=None,
                  exclude=None):
    """
    Download a given path on a cluster into an assetstore.

    :params cluster_connection: The cluster connection to access the cluster.
    :params girder_token: The Girder token to use to access Girder.
    :params parent: The target folder to import the path into.
    :params path: The path on the cluster to download.
    :params assetstore_url: The url for the assetstore to use for the import.
    :params assetstore_id: The id of the asseststore to import into.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    """
    girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
    girder_client.token = girder_token

    _import_path(cluster_connection, girder_client, parent, path,
                 assetstore_url, assetstore_id, upload=upload, include=include,
                 exclude=exclude)

def download_path_from_cluster(cluster, girder_token, parent, path,
                               upload=False, include=None, exclude=None):
    """
    Download a given path on a cluster into an assetstore.

    :params cluster: The cluster to to download the path from.
    :params girder_token: The Girder token to use to access Girder.
    :params parent: The target folder to import the path into.
    :params path: The path on the cluster to download.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    """
    assetstore_base_url = get_assetstore_url_base(cluster)
    assetstore_id = get_assetstore_id(girder_token, cluster)

    with get_connection(girder_token, cluster) as conn:
        download_path(conn, girder_token, parent, path, assetstore_base_url,
                      assetstore_id, upload=upload, include=include,
                      exclude=exclude)
<|endoftext|>


<commit_before># Build the executables that are used in the flopy autotests
import os
import sys
import shutil

try:
    import pymake
except:
    print('pymake is not installed...will not build executables')
    pymake = None

os.environ["TRAVIS"] = "1"

download_version = '3.0'

# path where downloaded executables will be extracted
exe_pth = 'exe_download'
# make the directory if it does not exist
if not os.path.isdir(exe_pth):
    os.makedirs(exe_pth)

# determine if running on Travis
is_travis = 'TRAVIS' in os.environ

bindir = '.'
dotlocal = False
if is_travis:
    dotlocal = True

if not dotlocal:
    for idx, arg in enumerate(sys.argv):
        if '--travis' in arg.lower():
            dotlocal = True
            break
if dotlocal:
    bindir = os.path.join(os.path.expanduser('~'), '.local', 'bin')
    bindir = os.path.abspath(bindir)

# write where the executables will be downloaded
print('modflow executables will be downloaded to:\n\n    "{}"'.format(bindir))

build_from_source = False
if not build_from_source:
    for idx, arg in enumerate(sys.argv):
        if '--build' in arg.lower():
            build_from_source = True
            break

# write if the executables will be built from source
msg = 'build all executables from source code: {}\n'.format(build_from_source)
print(msg)

def get_targets():
    targets = pymake.usgs_program_data.get_keys(current=True)
    targets.sort()
    targets.remove('vs2dt')
    return targets

def build_target(target):
    if pymake is not None:
        pymake.build_apps(targets=target)

    return

def which(program):
    """
    Test to make sure that the program is executable

    """
    import os
    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None

def get_modflow_exes(pth='.', version='', platform=None):
    """
    Get the latest MODFLOW binary executables from a github site
    (https://github.com/MODFLOW-USGS/executables) for the specified
    operating system and put them in the specified path.

    Parameters
    ----------
    pth : str
        Location to put the executables (default is current working directory)

    version : str
        Version of the MODFLOW-USGS/executables release to use.

    platform : str
        Platform that will run the executables.  Valid values include mac,
        linux, win32 and win64.  If platform is None, then routine will
        download the latest asset from the github reposity.

    """

    # Determine the platform in order to construct the zip file name
    if platform is None:
        if sys.platform.lower() == 'darwin':
            platform = 'mac'
        elif sys.platform.lower().startswith('linux'):
            platform = 'linux'
        elif 'win' in sys.platform.lower():
            is_64bits = sys.maxsize > 2 ** 32
            if is_64bits:
                platform = 'win64'
            else:
                platform = 'win32'
        else:
            errmsg = ('Could not determine platform'
                      '.  sys.platform is {}'.format(sys.platform))
            raise Exception(errmsg)
    else:
        assert platform in ['mac', 'linux', 'win32', 'win64']
    zipname = '{}.zip'.format(platform)

    # Determine path for file download and then download and unzip
    url = ('https://github.com/MODFLOW-USGS/executables/'
           'releases/download/{}/'.format(version))
    assets = {p: url + p for p in ['mac.zip', 'linux.zip',
                                   'win32.zip', 'win64.zip']}
    download_url = assets[zipname]
    pymake.download_and_unzip(download_url, pth)

    return

def get_exes_list():
    temp = os.listdir(exe_pth)
    avail_targets = pymake.usgs_program_data.get_keys(current=True)
    targets = []
    for target in avail_targets:
        for tt in temp:
            if target in tt:
                targets.append(tt)

    # write summary of available download executables
    msg = 'executables that are available from the download:\n'
    for idx, target in enumerate(targets):
        msg += '    {:>2d}: {}\n'.format(idx + 1, target)
    print('{}\n'.format(msg))

    return targets

def is_executable(f):
    # write message
    msg = 'testing if {} is executable.'.format(f)
    print(msg)

    fname = os.path.join(exe_pth, f)
    errmsg = '{} not executable'.format(fname)
    assert which(fname) is not None, errmsg
    return

def get_code_json():
    jpth = 'code.json'
    json_dict = None
    if jpth in os.listdir(exe_pth):
        fpth = os.path.join(exe_pth, jpth)
        json_dict = pymake.usgs_program_data.load_json(fpth)

    return json_dict

def evaluate_versions(target, src):
    # get code.json dictionary
    json_dict = get_code_json()

    # get current modflow program dictionary
    prog_dict = pymake.usgs_program_data.get_program_dict()

    if json_dict is not None:
        # extract the json keys
        json_keys = list(json_dict.keys())
        # evaluate if the target is in the json keys
        if target in json_keys:
            source_version = prog_dict[target].version
            git_version = json_dict[target].version

            # write a message
            msg = 'Source code version of {} '.format(target) + \
                  'is "{}"'.format(source_version)
            print(4 * ' ' + msg)
            msg = 'Download code version of {} '.format(target) + \
                  'is "{}"\n'.format(git_version)
            print(4 * ' ' + msg)

            prog_version = source_version.split('.')
            json_version = git_version.split('.')

            # evaluate major, minor, etc. version numbers
            for sp, sj in zip(prog_version, json_version):
                if int(sp) > int(sj):
                    src = None
                    break

    return src

def copy_target(target, src):
    srcpth = os.path.join(exe_pth, src)
    dstpth = os.path.join(bindir, src)

    # write message showing copy src and dst
    msg = 'copying {} -> {}'.format(srcpth, dstpth)
    print(msg)

    # copy the target
    shutil.copy(srcpth, dstpth)

    # determine if json file in directory with downloaded files
    json_file = None
    for f in os.listdir(exe_pth):
        if f.lower().endswith('.json'):
            json_file = f
            break

    # update code.json with data from from the json file in exe_pth
    if json_file is not None:
        fpth = os.path.join(bindir, json_file)

        # set default program dictionary
        usgs_dict = pymake.usgs_program_data.get_program_dict()
        target_dict = {target: usgs_dict[target]}

        # process the json
        download_dict = pymake.usgs_program_data.load_json(fpth)
        if download_dict is not None:
            if target in list(download_dict.keys()):
                target_dict = {target: download_dict[target]}

        # update the json
        pymake.usgs_program_data.update_json(fpth=fpth,
                                             temp_dict=target_dict)

    return

def list_json():
    # build list_json command
    fpth = os.path.join(bindir, 'code.json')
    ljson = 'pymake.usgs_program_data.list_json(fpth="{}")'.format(fpth)

    # build full command
    cmd = "python -c 'from __future__ import print_function; " + \
          "import pymake; {}'".format(ljson)

    # run command
    os.system(cmd)
    return

def cleanup():
    if os.path.isdir(exe_pth):
        shutil.rmtree(exe_pth)

    return

def main():
    get_modflow_exes(exe_pth, download_version)

    etargets = get_exes_list()
    for f in etargets:
        is_executable(f)

    # build each target
    targets = get_targets()
    for etarget in etargets:
        src = None
        for target in targets:
            if target in etarget:
                src = etarget
                break

        # evaluate if the usgs source files and newer versions than
        # downloaded executables...if so build the target from source code
        if src is not None:
            src = evaluate_versions(target, src)

        # reset source if code should be rebuilt from source
        if build_from_source:
            src = None

        # copy the downloaded executable
        if src is not None:
            copy_target(target, src)
        # build the target from source code
        else:
            msg = 'building {}'.format(target)
            print(msg)
            build_target(target)

        # # build all targets (until github gfortran-8 exes are available)
        # build_target(target)

    # list the created json file
    list_json()

    # clean up the download directory
    cleanup()

def test_download_and_unzip():
    yield get_modflow_exes, exe_pth, download_version

    etargets = get_exes_list()
    for f in etargets:
        yield is_executable, f
    return

def test_build_all_apps():
    # get list of downloaded targets
    etargets = get_exes_list()

    # build each target
    targets = get_targets()
    for etarget in etargets:
        src = None
        for target in targets:
            if target in etarget:
                src = etarget
                break

        # evaluate if the usgs source files and newer versions than
        # downloaded executables...if so build the target from source code
        if src is not None:
            src = evaluate_versions(target, src)

        # reset source if code should be rebuilt from source
        if build_from_source:
            src = None

        # copy the downloaded executable
        if src is not None:
            yield copy_target, target, src
        # build the target
        else:
            msg = 'building {}'.format(target)
            print(msg)
            yield build_target, target

        # # build all targets (until github gfortran-8 exes are available)
        # yield build_target, target

    return

def test_list_json():
    list_json()
    return

def test_cleanup():
    cleanup()

if __name__ == '__main__':
    main()
<commit_msg>fix(build_exes.py): fix bugs for Windows (#810) (#813)<commit_after># Build the executables that are used in the flopy autotests
import os
import sys
import shutil

try:
    import pymake
except:
    print('pymake is not installed...will not build executables')
    pymake = None

os.environ["TRAVIS"] = "1"

download_version = '3.0'

# path where downloaded executables will be extracted
exe_pth = 'exe_download'
# make the directory if it does not exist
if not os.path.isdir(exe_pth):
    os.makedirs(exe_pth)

# determine if running on Travis
is_travis = 'TRAVIS' in os.environ

bindir = '.'
dotlocal = False
if is_travis:
    dotlocal = True

if not dotlocal:
    for idx, arg in enumerate(sys.argv):
        if '--travis' in arg.lower():
            dotlocal = True
            break
if dotlocal:
    bindir = os.path.join(os.path.expanduser('~'), '.local', 'bin')
    bindir = os.path.abspath(bindir)
    if not os.path.isdir(bindir):
        os.makedirs(bindir)

# write where the executables will be downloaded
print('modflow executables will be downloaded to:\n\n    "{}"'.format(bindir))

build_from_source = False
if not build_from_source:
    for idx, arg in enumerate(sys.argv):
        if '--build' in arg.lower():
            build_from_source = True
            break

# write if the executables will be built from source
msg = 'build all executables from source code: {}\n'.format(build_from_source)
print(msg)

def get_targets():
    targets = pymake.usgs_program_data.get_keys(current=True)
    targets.sort()
    targets.remove('vs2dt')
    return targets

def build_target(target):
    if pymake is not None:
        pymake.build_apps(targets=target)

    return

def which(program):
    """
    Test to make sure that the program is executable

    """
    import os
    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None

def get_modflow_exes(pth='.', version='', platform=None):
    """
    Get the latest MODFLOW binary executables from a github site
    (https://github.com/MODFLOW-USGS/executables) for the specified
    operating system and put them in the specified path.

    Parameters
    ----------
    pth : str
        Location to put the executables (default is current working directory)

    version : str
        Version of the MODFLOW-USGS/executables release to use.

    platform : str
        Platform that will run the executables.  Valid values include mac,
        linux, win32 and win64.  If platform is None, then routine will
        download the latest asset from the github reposity.

    """

    # Determine the platform in order to construct the zip file name
    if platform is None:
        if sys.platform.lower() == 'darwin':
            platform = 'mac'
        elif sys.platform.lower().startswith('linux'):
            platform = 'linux'
        elif 'win' in sys.platform.lower():
            is_64bits = sys.maxsize > 2 ** 32
            if is_64bits:
                platform = 'win64'
            else:
                platform = 'win32'
        else:
            errmsg = ('Could not determine platform'
                      '.  sys.platform is {}'.format(sys.platform))
            raise Exception(errmsg)
    else:
        assert platform in ['mac', 'linux', 'win32', 'win64']
    zipname = '{}.zip'.format(platform)

    # Determine path for file download and then download and unzip
    url = ('https://github.com/MODFLOW-USGS/executables/'
           'releases/download/{}/'.format(version))
    assets = {p: url + p for p in ['mac.zip', 'linux.zip',
                                   'win32.zip', 'win64.zip']}
    download_url = assets[zipname]
    pymake.download_and_unzip(download_url, pth)

    return

def get_exes_list():
    temp = os.listdir(exe_pth)
    avail_targets = get_targets()
    targets = []
    for target in avail_targets:
        for tt in temp:
            if target in tt:
                targets.append(tt)

    # write summary of available download executables
    msg = 'executables that are available from the download:\n'
    for idx, target in enumerate(targets):
        msg += '    {:>2d}: {}\n'.format(idx + 1, target)
    print('{}\n'.format(msg))

    return targets

def is_executable(f):
    # write message
    msg = 'testing if {} is executable.'.format(f)
    print(msg)

    fname = os.path.join(exe_pth, f)
    errmsg = '{} not executable'.format(fname)
    assert which(fname) is not None, errmsg
    return

def get_code_json():
    jpth = 'code.json'
    json_dict = None
    if jpth in os.listdir(exe_pth):
        fpth = os.path.join(exe_pth, jpth)
        json_dict = pymake.usgs_program_data.load_json(fpth)

    return json_dict

def evaluate_versions(target, src):
    # get code.json dictionary
    json_dict = get_code_json()

    # get current modflow program dictionary
    prog_dict = pymake.usgs_program_data.get_program_dict()

    if json_dict is not None:
        # extract the json keys
        json_keys = list(json_dict.keys())
        # evaluate if the target is in the json keys
        if target in json_keys:
            source_version = prog_dict[target].version
            git_version = json_dict[target].version

            # write a message
            msg = 'Source code version of {} '.format(target) + \
                  'is "{}"'.format(source_version)
            print(4 * ' ' + msg)
            msg = 'Download code version of {} '.format(target) + \
                  'is "{}"\n'.format(git_version)
            print(4 * ' ' + msg)

            prog_version = source_version.split('.')
            json_version = git_version.split('.')

            # evaluate major, minor, etc. version numbers
            for sp, sj in zip(prog_version, json_version):
                if int(sp) > int(sj):
                    src = None
                    break

    return src

def copy_target(target, src):
    srcpth = os.path.join(exe_pth, src)
    dstpth = os.path.join(bindir, src)

    # write message showing copy src and dst
    msg = 'copying {} -> {}'.format(srcpth, dstpth)
    print(msg)

    # copy the target
    shutil.copy(srcpth, dstpth)

    # determine if json file in directory with downloaded files
    json_file = None
    for f in os.listdir(exe_pth):
        if f.lower().endswith('.json'):
            json_file = f
            break

    # update code.json with data from from the json file in exe_pth
    if json_file is not None:
        fpth = os.path.join(bindir, json_file)

        # set default program dictionary
        usgs_dict = pymake.usgs_program_data.get_program_dict()
        target_dict = {target: usgs_dict[target]}

        # process the json
        download_dict = pymake.usgs_program_data.load_json(fpth)
        if download_dict is not None:
            if target in list(download_dict.keys()):
                target_dict = {target: download_dict[target]}

        # update the json
        pymake.usgs_program_data.update_json(fpth=fpth,
                                             temp_dict=target_dict)

    return

def list_json():
    # determine if json file in directory with downloaded files
    json_file = None
    for f in os.listdir(exe_pth):
        if f.lower().endswith('.json'):
            json_file = f
            break

    if json_file is not None:
        # get shell syntax utilities
        import shlex
        shlex_obj = shlex.shlex()
        e = shlex_obj.escape
        q = shlex_obj.quotes
        eq = shlex_obj.escapedquotes

        # build list_json command
        fpth = os.path.abspath(os.path.join(bindir, 'code.json'))
        if os.sep in e:
            fpth = fpth.replace(os.sep, e[0] + os.sep)
        quoted_fpth = '"' + fpth + '"'
        if '"' in q:
            quoted_fpth = quoted_fpth.replace('"', e[0] + '"')
        ljson = 'pymake.usgs_program_data.list_json(fpth=' + quoted_fpth + ')'

        # build full command
        cmd = 'python -c ' + eq[0] + 'from __future__ import print_function; ' + \
                  'import pymake; ' + ljson + eq[0]

        # run command
        os.system(cmd)

    return

def cleanup():
    if os.path.isdir(exe_pth):
        shutil.rmtree(exe_pth)

    return

def main():
    get_modflow_exes(exe_pth, download_version)

    etargets = get_exes_list()
    for f in etargets:
        is_executable(f)

    # build each target
    targets = get_targets()
    for etarget in etargets:
        src = None
        for target in targets:
            if target in etarget:
                src = etarget
                break

        # evaluate if the usgs source files and newer versions than
        # downloaded executables...if so build the target from source code
        if src is not None:
            src = evaluate_versions(target, src)

        # reset source if code should be rebuilt from source
        if build_from_source:
            src = None

        # copy the downloaded executable
        if src is not None:
            copy_target(target, src)
        # build the target from source code
        else:
            msg = 'building {}'.format(target)
            print(msg)
            build_target(target)

        # # build all targets (until github gfortran-8 exes are available)
        # build_target(target)

    # list the created json file
    list_json()

    # clean up the download directory
    cleanup()

def test_download_and_unzip():
    yield get_modflow_exes, exe_pth, download_version

    etargets = get_exes_list()
    for f in etargets:
        yield is_executable, f
    return

def test_build_all_apps():
    # get list of downloaded targets
    etargets = get_exes_list()

    # build each target
    targets = get_targets()
    for etarget in etargets:
        src = None
        for target in targets:
            if target in etarget:
                src = etarget
                break

        # evaluate if the usgs source files and newer versions than
        # downloaded executables...if so build the target from source code
        if src is not None:
            src = evaluate_versions(target, src)

        # reset source if code should be rebuilt from source
        if build_from_source:
            src = None

        # copy the downloaded executable
        if src is not None:
            yield copy_target, target, src
        # build the target
        else:
            msg = 'building {}'.format(target)
            print(msg)
            yield build_target, target

        # # build all targets (until github gfortran-8 exes are available)
        # yield build_target, target

    return

def test_list_json():
    list_json()
    return

def test_cleanup():
    cleanup()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django import template
from django.template.defaultfilters import stringfilter

from classytags.core import Tag, Options
from classytags.arguments import Argument

from ..models import Currency
from ..utils import get_currency_code, calculate

register = template.Library()

class CurrentCurrencyTag(Tag):
    name = 'get_current_currency'
    options = Options(
        'as',
        Argument('varname', resolve=False, required=False),
    )

    def render_tag(self, context, varname):
        code = get_currency_code(context['request'])
        try:
            currency = Currency.active.get(code=code)
        except Currency.DoesNotExist:
            try:
                currency = Currency.active.default()
            except Currency.DoesNotExist:
                currency = None  # shit happens...

        if varname:
            context[varname] = currency
            return ''
        else:
            return currency

register.tag(CurrentCurrencyTag)

class ChangeCurrencyTag(Tag):
    name = 'change_currency'
    options = Options(
        Argument('price', resolve=True, required=True),
        Argument('code', resolve=True, required=True),
    )

    def render_tag(self, context, price, code):
        return calculate(price, code)

register.tag(ChangeCurrencyTag)

@stringfilter
@register.filter(name='currency')
def do_currency(price, code):
    return calculate(price, code)
<commit_msg>remove custom template tag<commit_after># -*- coding: utf-8 -*-

from django import template
from django.template.defaultfilters import stringfilter

from classytags.core import Tag, Options
from classytags.arguments import Argument

from ..models import Currency
from ..utils import get_currency_code, calculate

register = template.Library()

    options = Options(
        'as',
        Argument('varname', resolve=False, required=False),
    )

    def render_tag(self, context, varname):
        if varname:
            return ''
        else:

class ChangeCurrencyTag(Tag):
    name = 'change_currency'
    options = Options(
        Argument('price', resolve=True, required=True),
        Argument('code', resolve=True, required=True),
    )

    def render_tag(self, context, price, code):
        return calculate(price, code)

register.tag(ChangeCurrencyTag)

@stringfilter
@register.filter(name='currency')
def do_currency(price, code):
    return calculate(price, code)
<|endoftext|>


<commit_before>import shutilwhich  # noqa: F401
from shutil import which
import subprocess
import pysam

class BamAlignmentWriter(object):
    """Wrap pysam.AlignmentFile with sambamba for multithreaded compressed writing."""

    def __init__(self, path, template=None, header=None, threads=2, external_bin='choose_best'):
        """Use this class with a contexthandler."""
        self.template = template
        self.header = header
        self.path = path
        self.external_bin = external_bin
        self.threads = threads
        self.args = self.get_subprocess_args()

    def get_subprocess_args(self):
        """Figure out in sambamba or samtools are available and return correct arguments."""
        sambamba_args = ['sambamba', 'view', '-S', '-f', 'bam', '-t', "%s" % self.threads, '/dev/stdin', '-o', self.path]
        samtools_args = ['samtools', 'view', '-b', '/dev/stdin/', '-o', self.path]
        if self.external_bin == 'sambamba':
            return sambamba_args
        elif self.external_bin == 'samtools':
            return samtools_args
        elif self.external_bin == 'choose_best':
            if which('sambamba'):
                return sambamba_args
            if which('samtools'):
                return samtools_args
        return None

    def close(self):
        """Close filehandles and suprocess safely."""
        self.af.close()
        if self.args:
            self.proc.stdin.close()

    def __enter__(self):
        """Provide context handler entry."""
        if self.args:
            self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE)
            self.af = pysam.AlignmentFile(self.proc.stdin, mode="wh", template=self.template, header=self.header)
        else:
            self.af = pysam.AlignmentFile(self.path, mode="wb", template=self.template, header=self.header)
        return self.af

    def __exit__(self, type, value, traceback):
        """Provide context handler exit."""
        self.close()

class BamAlignmentReader(object):
    """Wraps pysam.AlignmentFile with sambamba for reading if input file is a bam file."""

    def __init__(self, path, external_bin='choose_best'):
        """Use this class with a contexthandler."""
        self.bam = pysam.AlignmentFile(path).is_bam
        self.path = path
        self.external_bin = external_bin
        self.args = self.get_subprocess_args()

    def get_subprocess_args(self):
        """Figure out in sambamba or samtools are available and return correct arguments."""
        sambamba_args = ['sambamba', 'view', '-h', self.path]
        samtools_args = ['samtools', 'view', '-h', self.path]
        if self.external_bin == 'sambamba':
            return sambamba_args
        elif self.external_bin == 'samtools':
            return samtools_args
        elif self.external_bin == 'choose_best':
            if which('sambamba'):
                return sambamba_args
            if which('samtools'):
                return samtools_args
        return None

    def close(self):
        """Close filehandles and suprocess safely."""
        self.af.close()
        if self.bam and self.args:
            self.proc.stdout.close()

    def __enter__(self):
        """Provide context handler entry."""
        if self.bam and self.args:
            self.proc = subprocess.Popen(self.args, stdout=subprocess.PIPE)
            self.af = pysam.AlignmentFile(self.proc.stdout)
        else:
            self.af = pysam.AlignmentFile(self.path)
        return self.af

    def __exit__(self, type, value, traceback):
        """Provide context handler exit."""
        self.close()
<commit_msg>Fix a bug where closing the fileobject would corrupt the BAM file<commit_after>import shutilwhich  # noqa: F401
from shutil import which
import subprocess
import pysam

class BamAlignmentWriter(object):
    """Wrap pysam.AlignmentFile with sambamba for multithreaded compressed writing."""

    def __init__(self, path, template=None, header=None, threads=2, external_bin='choose_best'):
        """Use this class with a contexthandler."""
        self.template = template
        self.header = header
        self.path = path
        self.external_bin = external_bin
        self.threads = threads
        self.args = self.get_subprocess_args()

    def get_subprocess_args(self):
        """Figure out in sambamba or samtools are available and return correct arguments."""
        sambamba_args = ['sambamba', 'view', '-f', 'bam', '-t', "%s" % self.threads, '/dev/stdin', '-o', self.path]
        samtools_args = ['samtools', 'view', '-b', '/dev/stdin/', '-o', self.path]
        if self.external_bin == 'sambamba':
            return sambamba_args
        elif self.external_bin == 'samtools':
            return samtools_args
        elif self.external_bin == 'choose_best':
            if which('sambamba'):
                return sambamba_args
            if which('samtools'):
                return samtools_args
        return None

    def close(self):
        """Close filehandles and suprocess safely."""
        self.af.close()
        if self.args:
            self.proc.stdin.close()

    def __enter__(self):
        """Provide context handler entry."""
        if self.args:
            self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE)
            self.af = pysam.AlignmentFile(self.proc.stdin, mode="wbu", template=self.template, header=self.header)
        else:
            self.af = pysam.AlignmentFile(self.path, mode="wb", template=self.template, header=self.header)
        return self.af

    def __exit__(self, type, value, traceback):
        """Provide context handler exit."""
        self.close()

class BamAlignmentReader(object):
    """Wraps pysam.AlignmentFile with sambamba for reading if input file is a bam file."""

    def __init__(self, path, external_bin='choose_best'):
        """Use this class with a contexthandler."""
        self.bam = pysam.AlignmentFile(path).is_bam
        self.path = path
        self.external_bin = external_bin
        self.args = self.get_subprocess_args()

    def get_subprocess_args(self):
        """Figure out in sambamba or samtools are available and return correct arguments."""
        sambamba_args = ['sambamba', 'view', '-h', self.path]
        samtools_args = ['samtools', 'view', '-h', self.path]
        if self.external_bin == 'sambamba':
            return sambamba_args
        elif self.external_bin == 'samtools':
            return samtools_args
        elif self.external_bin == 'choose_best':
            if which('sambamba'):
                return sambamba_args
            if which('samtools'):
                return samtools_args
        return None

    def close(self):
        """Close filehandles and suprocess safely."""
        self.af.close()
        if self.bam and self.args:
            self.proc.stdout.close()

    def __enter__(self):
        """Provide context handler entry."""
        if self.bam and self.args:
            self.proc = subprocess.Popen(self.args, stdout=subprocess.PIPE)
            self.af = pysam.AlignmentFile(self.proc.stdout)
        else:
            self.af = pysam.AlignmentFile(self.path)
        return self.af

    def __exit__(self, type, value, traceback):
        """Provide context handler exit."""
        self.close()
<|endoftext|>


<commit_before>import os
import sys
from shutil import copy2

class PrivilegeException(Exception):
    def __init__(self, value):
        self.paramater = value
    def __str__(self):
        return str(self.paramater)

PIP_STRING = \
"""pip must be installed for this script to function!

debian-based:   apt install python3-pip
arch-based:     pacman -S python-pip
redhat-based:   yum install python-pip

If not listed, check your favorite search engine."""

def main():
    if os.geteuid() != 0:
        raise PrivilegeException("You must have root privileges to run this install script.")
    if sys.version_info[0] < 3:
        raise Exception("Script must be run with python 3")

    print("Checking dependencies...")
    try:
        import colorama
    except ImportError:
        try:
            import pip
        except ImportError:
            print(PIP_STRING)
            exit(-1)

        def install(package):
            pip.main(['install', package])

        print("Required module [colorama] was not found. Installing via pip.")
        install('colorama')

    print("Copying files...")
    path = '/opt/wr'
    os.mkdir(path)
    copy2('./main.py', path)
    copy2('./joblib.py', path)
    copy2('./auxiliary.py', path)

    print("Creating symlink...")
    os.symlink('/opt/wr/main.py', '/usr/local/bin/wr')
    print("Successfully installed!")

if __name__ == '__main__':
    main()
<commit_msg>added stuff that removes pre-existing /opt/wr and /usr/local/bin/wr files<commit_after>import os
import sys
from shutil import copy2
from shutil import rmtree

class PrivilegeException(Exception):
    def __init__(self, value):
        self.paramater = value
    def __str__(self):
        return str(self.paramater)

PIP_STRING = \
"""pip must be installed for this script to function!

debian-based:   apt install python3-pip
arch-based:     pacman -S python-pip
redhat-based:   yum install python-pip

If not listed, check your favorite search engine."""

def main():
    if os.geteuid() != 0:
        raise PrivilegeException("You must have root privileges to run this install script.")
    if sys.version_info[0] < 3:
        raise Exception("Script must be run with python 3")

    print("Checking dependencies...")
    try:
        import colorama
    except ImportError:
        try:
            import pip
        except ImportError:
            print(PIP_STRING)
            exit(-1)

        def install(package):
            pip.main(['install', package])

        print("Required module [colorama] was not found. Installing via pip.")
        install('colorama')

    print("Copying files...")
    path = '/opt/wr'
    if os.path.exists(path):
        rmtree(path)

    os.mkdir(path)
    file_queue = ['./main.py', './joblib.py', './auxiliary.py']
    for file in file_queue:
        copy2(file, path)

    print("Creating symlink...")
    try: os.remove('/usr/local/bin/wr')
    except:
        pass
    os.symlink('/opt/wr/main.py', '/usr/local/bin/wr')

    print("Successfully installed!")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Script to combine predictions from different csvs to
slightly improve the log loss score. Will only work
if the models being combined all perform quite well.
"""

import os
import argparse
import neukrill_net.utils
import numpy as np
import json

def combine_csvs(csv_paths, verbose=False):
    """
    Takes two or more submission csv names and combines them by averaging the
    predictions in each. Returns this array of combined predictions.
    """

    # load each csv as an array, filling empty 3d array
    predictions = np.zeros((len(csv_paths),130400,121))
    for i,cpath in enumerate(csv_paths):
        # unfortunately, have to check the csv name is correct
        if cpath.split(".")[-1] != "gz":
            cpath = cpath + ".gz"
        if verbose:
            print("Loading {0}...".format(cpath))
        # have to enforce str otherwise loadtxt breaks (doesn't like unicode)
        predictions[i,:] = np.loadtxt(str(cpath), skiprows=1,
            usecols=range(1,122), delimiter=",")
    # average the arrays along the first axis
    if verbose:
        print("Averaging predictions...")
    predictions = np.mean(predictions, axis=0)
    return predictions

if __name__ == "__main__":
    # make a parser
    parser = argparse.ArgumentParser(description='Script to combine predictions'
            ' from different csvs to slightly improve the log loss score. Will'
            ' only work if the models being combined all perform quite well.')
    # run settings to combine
    parser.add_argument('run_settings', metavar='run_settings', type=str,
            nargs='+', default=os.path.join("run_settings","default.json"),
            help="Paths to run settings json files.")
    # output file path
    parser.add_argument('-f', nargs='?', help='Output csv file name.',
            type=str)
    # add verbose option
    parser.add_argument('-v', action="store_true", help="Run verbose.")
    args = parser.parse_args()
    # load the settings
    settings = neukrill_net.utils.Settings('settings.json')
    # get the csv paths
    csv_paths = []
    for rspath in args.run_settings:
        # load the run settings
        with open(rspath) as f:
            rs = json.load(f)
        csv_paths.append(rs["submissions abspath"])
    # pass the csv paths to the combiner to combine
    predictions = combine_csvs(csv_paths, verbose=args.v)
    # get the right filenames
    names = [os.path.basename(fpath) for fpath in settings.image_fnames['test']]
    if args.v:
        print("Writing new csv to {0}".format(args.f))
    # write the new csv to output file
    neukrill_net.utils.write_predictions(args.f, predictions,
            names, settings.classes)
<commit_msg>function now supports weights, should still operate as normal<commit_after>#!/usr/bin/env python
"""
Script to combine predictions from different csvs to
slightly improve the log loss score. Will only work
if the models being combined all perform quite well.
"""

import os
import argparse
import neukrill_net.utils
import numpy as np
import json

def combine_csvs(csv_paths, weights=None, verbose=False):
    """
    Takes two or more submission csv names and combines them by averaging the
    predictions in each. Returns this array of combined predictions.
    """
    # check if we're using weights
    if weights:
        # check weights sum to 1
        assert sum(weights) > 0.99999 and sum(weights) < 0.99999
    else:
        # make uniform weights
        weights = 1./len(csv_paths)

    # load each csv as an array, filling empty 3d array
    predictions = np.zeros((len(csv_paths),130400,121))
    for i,cpath in enumerate(csv_paths):
        # unfortunately, have to check the csv name is correct
        if cpath.split(".")[-1] != "gz":
            cpath = cpath + ".gz"
        if verbose:
            print("Loading {0}...".format(cpath))
        # have to enforce str otherwise loadtxt breaks (doesn't like unicode)
        predictions[i,:] = np.loadtxt(str(cpath), skiprows=1,
            usecols=range(1,122), delimiter=",")
    # average the arrays along the first axis
    if verbose:
        print("Averaging predictions...")
    # apply weights and sum
    for i in range(len(csv_paths)):
        predictions[i,:] = weights[i]*predictions[i,:]
    predictions = np.sum(predictions, axis=0)
    return predictions

if __name__ == "__main__":
    # make a parser
    parser = argparse.ArgumentParser(description='Script to combine predictions'
            ' from different csvs to slightly improve the log loss score. Will'
            ' only work if the models being combined all perform quite well.')
    # run settings to combine
    parser.add_argument('run_settings', metavar='run_settings', type=str,
            nargs='+', default=os.path.join("run_settings","default.json"),
            help="Paths to run settings json files.")
    # output file path
    parser.add_argument('-f', nargs='?', help='Output csv file name.',
            type=str)
    # add verbose option
    parser.add_argument('-v', action="store_true", help="Run verbose.")
    args = parser.parse_args()
    # load the settings
    settings = neukrill_net.utils.Settings('settings.json')
    # get the csv paths
    csv_paths = []
    for rspath in args.run_settings:
        # load the run settings
        with open(rspath) as f:
            rs = json.load(f)
        csv_paths.append(rs["submissions abspath"])
    # pass the csv paths to the combiner to combine
    predictions = combine_csvs(csv_paths, verbose=args.v)
    # get the right filenames
    names = [os.path.basename(fpath) for fpath in settings.image_fnames['test']]
    if args.v:
        print("Writing new csv to {0}".format(args.f))
    # write the new csv to output file
    neukrill_net.utils.write_predictions(args.f, predictions,
            names, settings.classes)
<|endoftext|>