<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import print_function

import fnmatch
import itertools
import json
import logging
import os
import re
import urlparse

import requests
import requests.exceptions
from kazoo.handlers.threading import KazooTimeoutError
from kazoo.retry import KazooRetry

from . import exceptions
from . import framework
from . import log
from . import mesos_file
from . import slave
from . import task
from . import util
from . import zookeeper
from paasta_tools.utils import retry

ZOOKEEPER_TIMEOUT = 1

INVALID_PATH = "{0} does not have a valid path. Did you forget /mesos?"

MISSING_MASTER = """unable to connect to a master at {0}.

Try running `mesos config master zk://localhost:2181/mesos`. See the README for
more examples."""

MULTIPLE_SLAVES = "There are multiple slaves with that id. Please choose one: "

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class MesosMaster(object):

    def __init__(self, config):
        self.config = config

    def __str__(self):
        return "<master: {0}>".format(self.key())

    def key(self):
        return self.config["master"]

    @util.CachedProperty(ttl=5)
    def host(self):
        return "{0}://{1}".format(self.config["scheme"], self.resolve(self.config["master"]))

    @log.duration
    def fetch(self, url, **kwargs):
        try:
            return requests.get(
                urlparse.urljoin(self.host, url),
                timeout=self.config["response_timeout"],
                **kwargs)
        except requests.exceptions.ConnectionError:
            raise exceptions.MasterNotAvailableException(MISSING_MASTER.format(self.host))

    def _file_resolver(self, cfg):
        return self.resolve(open(cfg[6:], "r+").read().strip())

    @retry(KazooTimeoutError, tries=5, delay=0.5, logger=logger)
    def _zookeeper_resolver(self, cfg):
        hosts, path = cfg[5:].split("/", 1)
        path = "/" + path

        retry = KazooRetry(max_tries=10)
        with zookeeper.client(hosts=hosts, read_only=True, connection_retry=retry) as zk:
            def master_id(key):
                return int(key.split("_")[-1])

            def get_masters():
                return [x for x in zk.get_children(path)
                        if re.search("\d+", x)]

            leader = sorted(get_masters(), key=lambda x: master_id(x))

            if len(leader) == 0:
                raise exceptions.MasterNotAvailableException("cannot find any masters at {0}".format(cfg,))
            data, stat = zk.get(os.path.join(path, leader[0]))

            if not data:
                exceptions.MasterNotAvailableException("Cannot retrieve valid MasterInfo data from ZooKeeper")

            try:
                parsed = json.loads(data)
                if parsed and "address" in parsed:
                    ip = parsed["address"].get("ip")
                    port = parsed["address"].get("port")
                    if ip and port:
                        return "{ip}:{port}".format(ip=ip, port=port)
            except ValueError as parse_error:
                log.debug("[WARN] No JSON content, probably connecting to older Mesos version. "
                          "Reason: {}".format(parse_error))
                raise exceptions.MasterNotAvailableException("Failed to parse mesos master ip from ZK")

    @log.duration
    def resolve(self, cfg):
        """Resolve the URL to the mesos master.

        The value of cfg should be one of:
            - host:port
            - zk://host1:port1,host2:port2/path
            - zk://username:password@host1:port1/path
            - file:///path/to/file (where file contains one of the above)
        """
        if cfg.startswith("zk:"):
            return self._zookeeper_resolver(cfg)
        elif cfg.startswith("file:"):
            return self._file_resolver(cfg)
        else:
            return cfg

    @util.CachedProperty(ttl=5)
    def state(self):
        return self.fetch("/master/state.json").json()

    def state_summary(self):
        return self.fetch("/master/state-summary").json()

    @util.memoize
    def slave(self, fltr):
        lst = self.slaves(fltr)

        log.debug("master.slave({0})".format(fltr))

        if len(lst) == 0:
            raise exceptions.SlaveDoesNotExist(
                "Slave {0} no longer exists.".format(fltr))

        elif len(lst) > 1:
            raise exceptions.MultipleSlavesForIDError(
                "Multiple slaves matching filter %s. %s" % (
                    fltr,
                    ",".join([slave.id for slave in lst])
                )
            )

        return lst[0]

    def slaves(self, fltr=""):
        return list(map(
            lambda x: slave.MesosSlave(self.config, x),
            itertools.ifilter(
                lambda x: fltr == x['id'], self.state['slaves'])))

    def _task_list(self, active_only=False):
        keys = ["tasks"]
        if not active_only:
            keys.append("completed_tasks")
        return itertools.chain(
            *[util.merge(x, *keys) for x in self._framework_list(active_only)])

    def task(self, fltr):
        lst = self.tasks(fltr)

        if len(lst) == 0:
            raise exceptions.TaskNotFoundException("Cannot find a task with filter %s" % fltr)

        elif len(lst) > 1:
            raise exceptions.MultipleTasksForIDError(
                "Multiple tasks matching filter %s. %s" % (
                    fltr,
                    ",".join([task.id for task in lst])
                )
            )
        return lst[0]

    # XXX - need to filter on task state as well as id
    def tasks(self, fltr="", active_only=False):
        return list(map(
            lambda x: task.Task(self, x),
            itertools.ifilter(
                lambda x: fltr in x["id"] or fnmatch.fnmatch(x["id"], fltr),
                self._task_list(active_only))))

    def framework(self, fwid):
        return list(filter(
            lambda x: x.id == fwid,
            self.frameworks()))[0]

    def _framework_list(self, active_only=False):
        keys = ["frameworks"]
        if not active_only:
            keys.append("completed_frameworks")
        return util.merge(self._frameworks, *keys)

    @util.CachedProperty(ttl=5)
    def _frameworks(self):
        return self.fetch("/master/frameworks").json()

    def frameworks(self, active_only=False):
        return [framework.Framework(f) for f in self._framework_list(active_only)]

    def metrics_snapshot(self):
        return self.fetch("/metrics/snapshot").json()

    @property
    @util.memoize
    def log(self):
        return mesos_file.File(self, path="/master/log")
<commit_msg>add command_retry to zk client<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import print_function

import fnmatch
import itertools
import json
import logging
import os
import re
import urlparse

import requests
import requests.exceptions
from kazoo.handlers.threading import KazooTimeoutError
from kazoo.retry import KazooRetry

from . import exceptions
from . import framework
from . import log
from . import mesos_file
from . import slave
from . import task
from . import util
from . import zookeeper
from paasta_tools.utils import retry

ZOOKEEPER_TIMEOUT = 1

INVALID_PATH = "{0} does not have a valid path. Did you forget /mesos?"

MISSING_MASTER = """unable to connect to a master at {0}.

Try running `mesos config master zk://localhost:2181/mesos`. See the README for
more examples."""

MULTIPLE_SLAVES = "There are multiple slaves with that id. Please choose one: "

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class MesosMaster(object):

    def __init__(self, config):
        self.config = config

    def __str__(self):
        return "<master: {0}>".format(self.key())

    def key(self):
        return self.config["master"]

    @util.CachedProperty(ttl=5)
    def host(self):
        return "{0}://{1}".format(self.config["scheme"], self.resolve(self.config["master"]))

    @log.duration
    def fetch(self, url, **kwargs):
        try:
            return requests.get(
                urlparse.urljoin(self.host, url),
                timeout=self.config["response_timeout"],
                **kwargs)
        except requests.exceptions.ConnectionError:
            raise exceptions.MasterNotAvailableException(MISSING_MASTER.format(self.host))

    def _file_resolver(self, cfg):
        return self.resolve(open(cfg[6:], "r+").read().strip())

    @retry(KazooTimeoutError, tries=5, delay=0.5, logger=logger)
    def _zookeeper_resolver(self, cfg):
        hosts, path = cfg[5:].split("/", 1)
        path = "/" + path

        retry = KazooRetry(max_tries=10)
        with zookeeper.client(hosts=hosts, read_only=True, connection_retry=retry, command_retry=retry) as zk:
            def master_id(key):
                return int(key.split("_")[-1])

            def get_masters():
                return [x for x in zk.get_children(path)
                        if re.search("\d+", x)]

            leader = sorted(get_masters(), key=lambda x: master_id(x))

            if len(leader) == 0:
                raise exceptions.MasterNotAvailableException("cannot find any masters at {0}".format(cfg,))
            data, stat = zk.get(os.path.join(path, leader[0]))

            if not data:
                exceptions.MasterNotAvailableException("Cannot retrieve valid MasterInfo data from ZooKeeper")

            try:
                parsed = json.loads(data)
                if parsed and "address" in parsed:
                    ip = parsed["address"].get("ip")
                    port = parsed["address"].get("port")
                    if ip and port:
                        return "{ip}:{port}".format(ip=ip, port=port)
            except ValueError as parse_error:
                log.debug("[WARN] No JSON content, probably connecting to older Mesos version. "
                          "Reason: {}".format(parse_error))
                raise exceptions.MasterNotAvailableException("Failed to parse mesos master ip from ZK")

    @log.duration
    def resolve(self, cfg):
        """Resolve the URL to the mesos master.

        The value of cfg should be one of:
            - host:port
            - zk://host1:port1,host2:port2/path
            - zk://username:password@host1:port1/path
            - file:///path/to/file (where file contains one of the above)
        """
        if cfg.startswith("zk:"):
            return self._zookeeper_resolver(cfg)
        elif cfg.startswith("file:"):
            return self._file_resolver(cfg)
        else:
            return cfg

    @util.CachedProperty(ttl=5)
    def state(self):
        return self.fetch("/master/state.json").json()

    def state_summary(self):
        return self.fetch("/master/state-summary").json()

    @util.memoize
    def slave(self, fltr):
        lst = self.slaves(fltr)

        log.debug("master.slave({0})".format(fltr))

        if len(lst) == 0:
            raise exceptions.SlaveDoesNotExist(
                "Slave {0} no longer exists.".format(fltr))

        elif len(lst) > 1:
            raise exceptions.MultipleSlavesForIDError(
                "Multiple slaves matching filter %s. %s" % (
                    fltr,
                    ",".join([slave.id for slave in lst])
                )
            )

        return lst[0]

    def slaves(self, fltr=""):
        return list(map(
            lambda x: slave.MesosSlave(self.config, x),
            itertools.ifilter(
                lambda x: fltr == x['id'], self.state['slaves'])))

    def _task_list(self, active_only=False):
        keys = ["tasks"]
        if not active_only:
            keys.append("completed_tasks")
        return itertools.chain(
            *[util.merge(x, *keys) for x in self._framework_list(active_only)])

    def task(self, fltr):
        lst = self.tasks(fltr)

        if len(lst) == 0:
            raise exceptions.TaskNotFoundException("Cannot find a task with filter %s" % fltr)

        elif len(lst) > 1:
            raise exceptions.MultipleTasksForIDError(
                "Multiple tasks matching filter %s. %s" % (
                    fltr,
                    ",".join([task.id for task in lst])
                )
            )
        return lst[0]

    # XXX - need to filter on task state as well as id
    def tasks(self, fltr="", active_only=False):
        return list(map(
            lambda x: task.Task(self, x),
            itertools.ifilter(
                lambda x: fltr in x["id"] or fnmatch.fnmatch(x["id"], fltr),
                self._task_list(active_only))))

    def framework(self, fwid):
        return list(filter(
            lambda x: x.id == fwid,
            self.frameworks()))[0]

    def _framework_list(self, active_only=False):
        keys = ["frameworks"]
        if not active_only:
            keys.append("completed_frameworks")
        return util.merge(self._frameworks, *keys)

    @util.CachedProperty(ttl=5)
    def _frameworks(self):
        return self.fetch("/master/frameworks").json()

    def frameworks(self, active_only=False):
        return [framework.Framework(f) for f in self._framework_list(active_only)]

    def metrics_snapshot(self):
        return self.fetch("/metrics/snapshot").json()

    @property
    @util.memoize
    def log(self):
        return mesos_file.File(self, path="/master/log")
<|endoftext|>


<commit_before>#
# Jasy - Web Tooling Framework
# Copyright 2010-2011 Sebastian Werner
#

import os, logging, copy, hashlib

import jasy.core.Variables

import jasy.cleaner.DeadCode
import jasy.cleaner.Unused

import jasy.Optimization

from jasy.core.MetaData import MetaData
from jasy.core.Permutation import getKeys
from jasy.core.Translation import hasText

from jasy.parser.Parser import parse
from jasy.process.Compressor import compress

aliases = {}

__all__ = ["Class", "Error"]

class Error(Exception):
    def __init__(self, inst, msg):
        self.__msg = msg
        self.__inst = inst

    def __str__(self):
        return "Error processing class %s: %s" % (self.__inst, self.__msg)

class Class():
    def __init__(self, path, project=None):
        self.__path = path
        self.__mtime = os.stat(path).st_mtime

        if project:
            self.__project = project
            self.__root = project.getClassPath()
            self.__package = project.getPackage()
            self.__cache = project.getCache()
            self.__localPath = os.path.relpath(path, project.getClassPath())
            self.__id = self.__localPath[:-3]
        else:
            self.__root = os.path.dirname(path)
            self.__package = ""
            self.__cache = Cache(self.__root)
            self.__id = os.path.filename(path)
            self.__localPath = path

        # This is by far slower and not the default but helps in specific project structures
        if project is None or project.isFuzzy():
            self.__name = self.getMeta().name
            if self.__name is None:
                raise Exception("Could not figure out fuzzy class name of: %s" % path)
        else:
            self.__name = self.__id.replace(os.sep, ".")
            if self.__package:
                self.__name = self.__package + "." + self.__name

    def getProject(self):
        """Returns the project which the class belongs to"""
        return self.__project

    def getId(self):
        """Returns a unique identify of the class. Typically as it is stored inside the project."""
        return self.__id

    def getName(self):
        """Returns the class name of the class based on the file name (default) or on the meta data (fuzzy)."""
        return self.__name

    # Map Python built-ins
    __repr__ = getName
    __str__ = getName

    def getPath(self):
        """Returns the exact position of the class file in the file system."""
        return self.__path

    def getLocalPath(self):
        """Returns the relative path inside the project (or the full path when no project is given)."""
        return self.__localPath

    def getModificationTime(self):
        """Returns last modification time of the class"""
        return self.__mtime

    def getText(self):
        """Reads the file (as UTF-8) and returns the text"""
        return open(self.__path, mode="r", encoding="utf-8").read()

    def getTree(self, permutation=None):
        """
        Returns the tree (of nodes from the parser) of the class. This parses the class,
        creates the tree, applies and optional permutation, scans for variables usage
        and puts the tree into the cache before returning it. The cache works with the
        permutation, so every permutated tree is cached separately.
        """

        permutation = self.filterPermutation(permutation)

        field = "tree[%s]-%s" % (self.__id, permutation)
        tree = self.__cache.read(field, self.__mtime)
        if tree is not None:
            return tree

        # Parse tree
        tree = parse(self.getText(), self.__id)

        # Apply permutation
        if permutation:
            permutation.patch(tree)

        # Remove dead code
        jasy.cleaner.DeadCode.cleanup(tree)

        # Scan for variable usage
        jasy.core.Variables.scan(tree)

        # Remove unused variables/functions
        jasy.cleaner.Unused.cleanup(tree)

        self.__cache.store(field, tree, self.__mtime, True)
        return tree

    def getDependencies(self, permutation=None, classes=None):
        """
        Returns a set of dependencies seen through the given list of known
        classes (ignoring all unknown items in original set). This method
        makes use of the meta data (see core/MetaData.py) and the statistics data
        (see getStats() in this class) which uses the data of the variable
        scanner in (core/Variables.py).
        """

        permutation = self.filterPermutation(permutation)

        meta = self.getMeta(permutation)
        stats = self.getStats(permutation)
        result = set()

        # Manually defined names/classes
        for name in meta.requires:
            if name != self.__id and name in classes:
                result.add(classes[name])

        # Globally modified names (mostly relevant when working without namespaces)
        for name in stats.shared:
            if name != self.__id and name in classes:
                result.add(classes[name])

        # Add classes from detected package access
        for package in stats.packages:
            if package in aliases:
                className = aliases[package]
                if className in classes:
                    result.add(classes[className])
                    continue

            orig = package
            while True:
                if package == self.__id:
                    break

                elif package in classes:
                    aliases[orig] = package
                    result.add(classes[package])
                    break

                else:
                    pos = package.rfind(".")
                    if pos == -1:
                        break

                    package = package[0:pos]

        return result

    def getStats(self, permutation=None):
        """
        """

        permutation = self.filterPermutation(permutation)

        field = "stats[%s]-%s" % (self.__id, permutation)
        stats = self.__cache.read(field, self.__mtime)
        if stats == None:
            stats = self.getTree(permutation).stats
            self.__cache.store(field, stats, self.__mtime)

        return stats

    def getMeta(self, permutation=None):
        permutation = self.filterPermutation(permutation)

        field = "meta[%s]-%s" % (self.__id, permutation)
        meta = self.__cache.read(field, self.__mtime)
        if meta == None:
            meta = MetaData(self.getTree(permutation))
            self.__cache.store(field, meta, self.__mtime)

        return meta

    def getPermutationKeys(self):
        field = "permutations[%s]" % (self.__id)
        result = self.__cache.read(field, self.__mtime)
        if result is None:
            result = getKeys(self.getTree())
            self.__cache.store(field, result, self.__mtime)

        return result

    def usesTranslation(self):
        field = "translation[%s]" % (self.__id)
        result = self.__cache.read(field, self.__mtime)
        if result == None:
            result = hasText(self.getTree())
            self.__cache.store(field, result, self.__mtime)

        return result

    def filterPermutation(self, permutation):
        if permutation:
            keys = self.getPermutationKeys()
            if keys:
                return permutation.filter(keys)

        return None

    def filterTranslation(self, translation):
        if translation and self.usesTranslation():
            return translation

        return None

    def getCompressed(self, permutation=None, translation=None, optimization=None, format=None):
        permutation = self.filterPermutation(permutation)
        translation = self.filterTranslation(translation)

        field = "compressed[%s]-%s-%s-%s-%s" % (self.__id, permutation, translation, optimization, format)
        field = hashlib.md5(field.encode("utf-8")).hexdigest()

        compressed = self.__cache.read(field, self.__mtime)
        if compressed == None:
            tree = self.getTree(permutation)

            if translation or optimization:
                tree = copy.deepcopy(tree)

                if translation:
                    translation.patch(tree)

                if optimization:
                    try:
                        optimization.apply(tree)
                    except jasy.Optimization.Error as error:
                        raise Error(self, "Could not compress class! %s" % error)

            compressed = compress(tree, format)
            self.__cache.store(field, compressed, self.__mtime)

        return compressed

        <commit_msg>Added cleanup flag and respect it for caching<commit_after>#
# Jasy - Web Tooling Framework
# Copyright 2010-2011 Sebastian Werner
#

import os, logging, copy, hashlib

import jasy.core.Variables

import jasy.cleaner.DeadCode
import jasy.cleaner.Unused

import jasy.Optimization

from jasy.core.MetaData import MetaData
from jasy.core.Permutation import getKeys
from jasy.core.Translation import hasText

from jasy.parser.Parser import parse
from jasy.process.Compressor import compress

aliases = {}

__all__ = ["Class", "Error"]

class Error(Exception):
    def __init__(self, inst, msg):
        self.__msg = msg
        self.__inst = inst

    def __str__(self):
        return "Error processing class %s: %s" % (self.__inst, self.__msg)

class Class():
    def __init__(self, path, project=None):
        self.__path = path
        self.__mtime = os.stat(path).st_mtime

        if project:
            self.__project = project
            self.__root = project.getClassPath()
            self.__package = project.getPackage()
            self.__cache = project.getCache()
            self.__localPath = os.path.relpath(path, project.getClassPath())
            self.__id = self.__localPath[:-3]
        else:
            self.__root = os.path.dirname(path)
            self.__package = ""
            self.__cache = Cache(self.__root)
            self.__id = os.path.filename(path)
            self.__localPath = path

        # This is by far slower and not the default but helps in specific project structures
        if project is None or project.isFuzzy():
            self.__name = self.getMeta().name
            if self.__name is None:
                raise Exception("Could not figure out fuzzy class name of: %s" % path)
        else:
            self.__name = self.__id.replace(os.sep, ".")
            if self.__package:
                self.__name = self.__package + "." + self.__name

    def getProject(self):
        """Returns the project which the class belongs to"""
        return self.__project

    def getId(self):
        """Returns a unique identify of the class. Typically as it is stored inside the project."""
        return self.__id

    def getName(self):
        """Returns the class name of the class based on the file name (default) or on the meta data (fuzzy)."""
        return self.__name

    # Map Python built-ins
    __repr__ = getName
    __str__ = getName

    def getPath(self):
        """Returns the exact position of the class file in the file system."""
        return self.__path

    def getLocalPath(self):
        """Returns the relative path inside the project (or the full path when no project is given)."""
        return self.__localPath

    def getModificationTime(self):
        """Returns last modification time of the class"""
        return self.__mtime

    def getText(self):
        """Reads the file (as UTF-8) and returns the text"""
        return open(self.__path, mode="r", encoding="utf-8").read()

    def getTree(self, permutation=None, cleanup=True):
        """
        Returns the tree (of nodes from the parser) of the class. This parses the class,
        creates the tree, applies and optional permutation, scans for variables usage
        and puts the tree into the cache before returning it. The cache works with the
        permutation, so every permutated tree is cached separately.
        """

        permutation = self.filterPermutation(permutation)

        field = "tree[%s]-%s-%s" % (self.__id, permutation, cleanup)
        tree = self.__cache.read(field, self.__mtime)
        if tree is not None:
            return tree

        # Parse tree
        tree = parse(self.getText(), self.__id)

        # Apply permutation
        if permutation:
            permutation.patch(tree)

        # Remove dead code
        if cleanup:
            jasy.cleaner.DeadCode.cleanup(tree)

        # Scan for variable usage
        jasy.core.Variables.scan(tree)

        # Remove unused variables/functions
        if cleanup:
            jasy.cleaner.Unused.cleanup(tree)

        self.__cache.store(field, tree, self.__mtime, True)
        return tree

    def getDependencies(self, permutation=None, classes=None):
        """
        Returns a set of dependencies seen through the given list of known
        classes (ignoring all unknown items in original set). This method
        makes use of the meta data (see core/MetaData.py) and the statistics data
        (see getStats() in this class) which uses the data of the variable
        scanner in (core/Variables.py).
        """

        permutation = self.filterPermutation(permutation)

        meta = self.getMeta(permutation)
        stats = self.getStats(permutation)
        result = set()

        # Manually defined names/classes
        for name in meta.requires:
            if name != self.__id and name in classes:
                result.add(classes[name])

        # Globally modified names (mostly relevant when working without namespaces)
        for name in stats.shared:
            if name != self.__id and name in classes:
                result.add(classes[name])

        # Add classes from detected package access
        for package in stats.packages:
            if package in aliases:
                className = aliases[package]
                if className in classes:
                    result.add(classes[className])
                    continue

            orig = package
            while True:
                if package == self.__id:
                    break

                elif package in classes:
                    aliases[orig] = package
                    result.add(classes[package])
                    break

                else:
                    pos = package.rfind(".")
                    if pos == -1:
                        break

                    package = package[0:pos]

        return result

    def getStats(self, permutation=None):
        """
        """

        permutation = self.filterPermutation(permutation)

        field = "stats[%s]-%s" % (self.__id, permutation)
        stats = self.__cache.read(field, self.__mtime)
        if stats == None:
            stats = self.getTree(permutation).stats
            self.__cache.store(field, stats, self.__mtime)

        return stats

    def getMeta(self, permutation=None):
        permutation = self.filterPermutation(permutation)

        field = "meta[%s]-%s" % (self.__id, permutation)
        meta = self.__cache.read(field, self.__mtime)
        if meta == None:
            meta = MetaData(self.getTree(permutation))
            self.__cache.store(field, meta, self.__mtime)

        return meta

    def getPermutationKeys(self):
        field = "permutations[%s]" % (self.__id)
        result = self.__cache.read(field, self.__mtime)
        if result is None:
            result = getKeys(self.getTree())
            self.__cache.store(field, result, self.__mtime)

        return result

    def usesTranslation(self):
        field = "translation[%s]" % (self.__id)
        result = self.__cache.read(field, self.__mtime)
        if result == None:
            result = hasText(self.getTree())
            self.__cache.store(field, result, self.__mtime)

        return result

    def filterPermutation(self, permutation):
        if permutation:
            keys = self.getPermutationKeys()
            if keys:
                return permutation.filter(keys)

        return None

    def filterTranslation(self, translation):
        if translation and self.usesTranslation():
            return translation

        return None

    def getCompressed(self, permutation=None, translation=None, optimization=None, format=None):
        permutation = self.filterPermutation(permutation)
        translation = self.filterTranslation(translation)

        field = "compressed[%s]-%s-%s-%s-%s" % (self.__id, permutation, translation, optimization, format)
        field = hashlib.md5(field.encode("utf-8")).hexdigest()

        compressed = self.__cache.read(field, self.__mtime)
        if compressed == None:
            tree = self.getTree(permutation)

            if translation or optimization:
                tree = copy.deepcopy(tree)

                if translation:
                    translation.patch(tree)

                if optimization:
                    try:
                        optimization.apply(tree)
                    except jasy.Optimization.Error as error:
                        raise Error(self, "Could not compress class! %s" % error)

            compressed = compress(tree, format)
            self.__cache.store(field, compressed, self.__mtime)

        return compressed

        <|endoftext|>


<commit_before># Topic server

import time
import flask

app = flask.Flask(__name__)
data = []
sessions = {}  # session ID -> current index  # FIXME No cleanup!

@app.route("/simple/newsession")
def new_session():
    session_id = "session" + hex(int(time.time() * 1000))[2:]
    sessions[session_id] = 0
    return session_id

@app.route("/simple/push/<session_id>/<value>")
def push(session_id, value):
    assert session_id in sessions
    data.append(value)
    return str(len(data) - 1)

@app.route("/simple/pop/<session_id>")
def pop(session_id):
    assert session_id in sessions
    if sessions[session_id] >= len(data):
        return ""
    res = data[sessions[session_id]]
    sessions[session_id] += 1
    return res

if __name__ == "__main__":
    app.run(host="127.0.0.1", port=8080)
<commit_msg>Return failure on pop() from an empty queue<commit_after># Topic server

import time
import flask

app = flask.Flask(__name__)
data = []
sessions = {}  # session ID -> current index  # FIXME No cleanup!

@app.route("/simple/newsession")
def new_session():
    session_id = "session" + hex(int(time.time() * 1000))[2:]
    sessions[session_id] = 0
    return session_id

@app.route("/simple/push/<session_id>/<value>")
def push(session_id, value):
    assert session_id in sessions     # Or crash, aka return error
    data.append(value)
    return str(len(data) - 1)

@app.route("/simple/pop/<session_id>")
def pop(session_id):
    assert session_id in sessions     # Or crash, aka return error
    res = data[sessions[session_id]]  # Or crash, aka return error
    sessions[session_id] += 1
    return res

if __name__ == "__main__":
    app.run(host="127.0.0.1", port=8080)
<|endoftext|>


<commit_before>'''List term operations.'''

import __builtin__ as builtins

from aterm import types
from aterm import visitor

# FIXME: write non-recursive versions

class _Operation(visitor.Visitor):
	'''Base visitor class for list operations.'''

	def visitTerm(self, term, *args, **kargs):
		raise TypeError('not a list term', term)

def empty(term):
	'''Whether a list term is empty or not.'''
	return types.isNil(term)

def length(term):
	'''Length of a list term.'''
	length = 0
	while not types.isNil(term):
		assert types.isCons(term)
		length += 1
		term = term.tail
	return length

def item(term, index):
	'''Get item at given index of a list term.'''
	if index < 0:
		raise IndexError('index out of bounds')
	while True:
		if types.isNil(term):
			raise IndexError('index out of bounds')
		if not types.isCons(term):
			raise TypeError('not a list term', term)
		if index == 0:
			return term.head
		index -= 1
		term = term.tail

class Iter(_Operation):
	'''List term iterator.'''

	def __init__(self, term):
		_Operation.__init__(self)
		self.term = term

	def next(self):
		return self.visit(self.term)

	def visitNil(self, term):
		raise StopIteration

	def visitCons(self, term):
		head = term.head
		self.term = term.tail
		return head

def extend(head, tail):
	'''Return the concatenation of two list terms.'''
	if types.isNil(head):
		return tail
	if types.isNil(tail):
		return head
	factory = tail.factory
	for elm in reversed(head):
		tail = factory.makeCons(elm, tail)
	return tail

def append(head, tail):
	'''Append an element to a list.'''
	factory = tail.factory
	return extend(head, factory.makeCons(other, factory.makeNil()))

class _Insert(_Operation):

	def visitNil(self, term, index, other):
		if index == 0:
			return term.factory.makeCons(other, term)
		else:
			raise IndexError('index out of bounds')

	def visitCons(self, term, index, other):
		if index == 0:
			return term.factory.makeCons(other, term)
		else:
			return term.factory.makeCons(
				term.head,
				self.visit(term.tail, index - 1, other),
			)

def insert(term, index, other):
	'''Insert an element into the list.'''
	return _Insert().visit(index, other)

def reverse(term):
	'''Reverse a list term.'''
	factory = term.factory
	accum = factory.makeNil()
	for elm in term:
		accum = factory.makeCons(elm, accum)
	return accum

def map(function, term):
	"""Return a list term with the elements for transformed by the function."""
	return term.factory.makeList(builtins.map(function, term))

def filter(function, term):
	"""Return a list term with the elements for which the function returns
	true."""
	return term.factory.makeList(builtins.filter(function, term))

class _Fetcher(_Operation):

	def __init__(self, function):
		_Operation.__init__(self)
		self.function = function

	def visitNil(self, term):
		return None

	def visitCons(self, term):
		if self.function(term.head):
			return term.head
		else:
			return self.visit(term.tail)

def fetch(function, term):
	"""Return a the first term of a list term for which the function returns
	true."""
	for elm in term:
		if function(elm):
			return elm
	return None

class _Splitter(visitor.Visitor):

	def __init__(self, index):
		visitor.Visitor.__init__(self)
		self.index = index

	def visitTerm(self, term, index):
		raise TypeError('not a term list', term)

	def visitNil(self, term, index):
		if index == self.index:
			return term.factory.makeNil(), term
		else:
			raise IndexError('index out of range')

	def visitCons(self, term, index):
		if index == self.index:
			return term.factory.makeNil(), term
		else:
			head, tail = self.visit(term.tail, index + 1)
			return term.factory.makeCons(
				term.head,
				head,
			), tail

def split(term, index):
	'''Splits a list term in two lists.
	The argument is the index of the first element of the second list.
	'''
	splitter = _Splitter(index)
	return splitter.visit(term, 0)
<commit_msg>Non-recursive implementation.<commit_after>'''List term operations.'''

from aterm import types
from aterm import visitor

def empty(term):
	'''Whether a list term is empty or not.'''
	return types.isNil(term)

def length(term):
	'''Length of a list term.'''
	length = 0
	while not types.isNil(term):
		assert types.isCons(term)
		length += 1
		term = term.tail
	return length

def item(term, index):
	'''Get item at given index of a list term.'''
	if index < 0:
		raise IndexError('index out of bounds')
	while True:
		if types.isNil(term):
			raise IndexError('index out of bounds')
		if not types.isCons(term):
			raise TypeError('not a list term', term)
		if index == 0:
			return term.head
		index -= 1
		term = term.tail

class Iter(visitor.Visitor):
	'''List term iterator.'''

	def __init__(self, term):
		visitor.Visitor.__init__(self)
		self.term = term

	def next(self):
		return self.visit(self.term)

	def visitTerm(self, term):
		raise TypeError('not a list term', term)

	def visitNil(self, term):
		raise StopIteration

	def visitCons(self, term):
		head = term.head
		self.term = term.tail
		return head

def extend(head, tail):
	'''Return the concatenation of two list terms.'''
	if types.isNil(head):
		return tail
	if types.isNil(tail):
		return head
	factory = tail.factory
	for elm in reversed(head):
		tail = factory.makeCons(elm, tail)
	return tail

def append(head, tail):
	'''Append an element to a list.'''
	factory = tail.factory
	return extend(head, factory.makeCons(other, factory.makeNil()))

def insert(term, index, other):
	'''Insert an element into the list.'''
	factory = term.factory
	accum = []
	for i in range(index):
		accum.append(term.head)
		term = term.tail
	term = factory.makeCons(other, term)
	for elem in reversed(accum):
		term = factory.makeCons(elem, term)
	return term

def reverse(term):
	'''Reverse a list term.'''
	factory = term.factory
	accum = factory.makeNil()
	for elm in term:
		accum = factory.makeCons(elm, accum)
	return accum

def map(function, term):
	"""Return a list term with the elements for transformed by the function."""
	return term.factory.makeList([function(elem) for elem in term])

def rmap(function, term):
	"""Return a list term with the elements for transformed by the function."""
	factory = term.factory
	accum = factory.makeNil()
	for elem in reversed(list(term)):
		accum = factory.makeCons(function(elem), accum)
	return accum

def filter(function, term):
	"""Return a list term with the elements for which the function returns
	true."""
	return term.factory.makeList([elem for elem in term if function(elem)])

def fetch(function, term):
	"""Return a the first term of a list term for which the function returns
	true."""
	for elm in term:
		if function(elm):
			return elm
	return None

def split(term, index):
	'''Splits a list term in two lists.
	The argument is the index of the first element of the second list.
	'''
	factory = term.factory
	head = []
	for i in range(index):
		head.append(term.head)
		term = term.tail
	return factory.makeList(head), term
<|endoftext|>


<commit_before>class DijkstraNew:
    def __init__(self,edges,start):
        self.edges = edges
        self.take_shorter_edges() # Bei doppelten Kanten kürzere nehmen
        self.start = start
        self.edges_in_dijkstra = [] # Kanten, über die Dijkstra geht
        self.visible_edges = [] # Sichtbare Kanten
        self.visible_nodes = [start] # Besuchte Knoten

    # Falls doppelte Kanten zwischen Knoten, dann nur kürzesten lassen
    def take_shorter_edges(self):
        delete_edges = self.index_of_longer_edges()
        if(delete_edges != [] and delete_edges != None):
            delete_edges.sort()
            delete_edges.reverse()
            self.delete_long_edges(delete_edges)

    # Indizes der langen Kanten bekommen
    def index_of_longer_edges(self):
        delete_edges = []
        for i in range(len(self.edges)):
            for j in range(len(self.edges)):
                if i != j and self.edges[i][0] == self.edges[j][0] and \
                 self.edges[i][2] == self.edges[j][2]:
                    if self.edges[i][1] > self.edges[j][1] \
                     and i not in delete_edges:
                        delete_edges.append(i)
                    elif self.edges[i][1] < self.edges[j][1] \
                     and j not in delete_edges:
                        delete_edges.append(j)
        return delete_edges

    # Lange Kanten löschen
    def delete_long_edges(self,delete_edges):
        for edge in delete_edges:
            self.edges.pop(edge)

    # Alle Kanten markieren, die sichtbar sind und noch nicht besucht wurden
    def get_visible_edges(self):
        self.visible_edges = []
        for edge in self.edges:
            if edge[0] in self.visible_nodes:
                self.visible_edges.append(edge)

    # Hauptfunktion in Dijkstra, gibt kürzesten Weg zu v2 aus
    def get_shortest_way(self,v2):

        while True:
            self.delete_unnecessary_edges()
            self.get_visible_edges()
<commit_msg>delete unnecessary edges and look for the shortest, visible edge<commit_after>class DijkstraNew:
    def __init__(self,edges,start):
        self.edges = edges
        self.take_shorter_edges() # Bei doppelten Kanten kürzere nehmen
        self.start = start
        self.edges_in_dijkstra = [] # Kanten, über die Dijkstra geht
        self.visible_edges = [] # Sichtbare Kanten
        self.visible_nodes = [start] # Besuchte Knoten

    # Falls doppelte Kanten zwischen Knoten, dann nur kürzesten lassen
    def take_shorter_edges(self):
        delete_edges = self.index_of_longer_edges()
        if(delete_edges != [] and delete_edges != None):
            delete_edges.sort()
            delete_edges.reverse()
            self.delete_long_edges(delete_edges)

    # Indizes der langen Kanten bekommen
    def index_of_longer_edges(self):
        delete_edges = []
        for i in range(len(self.edges)):
            for j in range(len(self.edges)):
                if i != j and self.edges[i][0] == self.edges[j][0] and \
                 self.edges[i][2] == self.edges[j][2]:
                    if self.edges[i][1] > self.edges[j][1] \
                     and i not in delete_edges:
                        delete_edges.append(i)
                    elif self.edges[i][1] < self.edges[j][1] \
                     and j not in delete_edges:
                        delete_edges.append(j)
        return delete_edges

    # Lange Kanten löschen
    def delete_long_edges(self,delete_edges):
        for edge in delete_edges:
            self.edges.pop(edge)

    # Alle Kanten markieren, die sichtbar sind und noch nicht besucht wurden
    def get_visible_edges(self):
        self.visible_edges = []
        for edge in self.edges:
            if edge[0] in self.visible_nodes:
                self.visible_edges.append(edge)

    # kürzeste, sichtbare Kante auswählen
    def get_shortest_edge(self):
        try:
            shortest_edge = self.visible_edges[0]# TODO try-catch, wenn Fehler, dann fertig
        except:
            return []
        for edge in self.visible_edges:
            if(shortest_edge[1] > edge[1]):
                shortest_edge = edge
        # Neuer Knoten erreichbar
        self.visible_nodes.append(shortest_edge[2])
        return shortest_edge

    #Löscht alle anderen Kanten, die zu schon besuchten Knoten gehen
    def delete_unnecessary_edges(self):
        delete_edges = []
        for edge in range(len(self.edges)):
            if(self.edges[edge][2] == self.visible_nodes[-1]):
                delete_edges.append(edge)
        if(delete_edges != [] and delete_edges != None):
            delete_edges.reverse()
            self.delete_long_edges(delete_edges)

    # Hauptfunktion in Dijkstra, gibt kürzesten Weg zu v2 aus
    def get_shortest_way(self,v2):

        while True:
            self.delete_unnecessary_edges()
            self.get_visible_edges()
            shortest_edge = self.get_shortest_edge()
            if(shortest_edge != []):
                self.edges_in_dijkstra.append(shortest_edge)
                self.visible_nodes.append(shortest_edge[2])
            else:
                break
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import argparse
import svgwrite

import random
import colorsys

def main(inputs, output):
    dwg = svgwrite.Drawing(output.name, profile='tiny')

    colors = get_n_colors(len(inputs))
    color_idx = 0

    parsed_inputs = parse_inputs(inputs)
    bounds = get_universal_bounds(parsed_inputs)

    for input in parsed_inputs:
        coords = points_to_coordinates(input["points"], bounds)
        dwg.add(dwg.polyline(coords, stroke=colors[color_idx], stroke_width="3", stroke_opacity="0.6", fill="none"))
        color_idx += 1

    dwg.save()

def parse_inputs(inputs):
    parsed_inputs = []
    for input in inputs:
        points = get_points_from_file(input)
        bounds = get_bounds(points)
        parsed_inputs.append({"points": points, "bounds": bounds})
    return parsed_inputs

def get_points_from_file(file):
    """ Parses a GPX file, and returns an array of all GPS points described by that file """
    parsed = BeautifulSoup(file.read(), 'html.parser')
    segments = parsed.trk.findAll("trkseg")
    points = []

    for segment in segments:
        segment_points = segment.findAll("trkpt")
        points.extend(map(lambda x: transform_bs4_point(x), segment_points))

    return points

def transform_bs4_point(bs4_point):
    """ Transforms a single BeautifulSoup node representing a node in the GPS coordinate graph
    into a dictionary representing the elements we care about """
    return {
        "lat": float(bs4_point["lat"]), # Degrees
        "lon": float(bs4_point["lon"]), # Degrees
        "ele": float(bs4_point.ele.string), # Meters?
        "time": bs4_point.time.string # ISO8601
    }

def get_universal_bounds(parsed_inputs):
    """ Gets the bounds of all recorded segments """
    min_lat = float("inf")
    max_lat = float("-inf")
    min_lon = float("inf")
    max_lon = float("-inf")

    for parsed_input in parsed_inputs:
        min_lat = parsed_input["bounds"]["min_lat"] if parsed_input["bounds"]["min_lat"] < min_lat else min_lat
        min_lon = parsed_input["bounds"]["min_lon"] if parsed_input["bounds"]["min_lon"] < min_lon else min_lon
        max_lat = parsed_input["bounds"]["max_lat"] if parsed_input["bounds"]["max_lat"] > max_lat else max_lat
        max_lon = parsed_input["bounds"]["max_lon"] if parsed_input["bounds"]["max_lon"] > max_lon else max_lon

    return {
        "min_lat": min_lat,
        "min_lon": min_lon,
        "max_lat": max_lat,
        "max_lon": max_lon
    }

def get_bounds(points):
    """ Returns the min and max lat and lon within an array of points """
    min_lat = float("inf")
    max_lat = float("-inf")
    min_lon = float("inf")
    max_lon = float("-inf")

    for point in points:
        min_lat = point["lat"] if point["lat"] < min_lat else min_lat
        min_lon = point["lon"] if point["lon"] < min_lon else min_lon
        max_lat = point["lat"] if point["lat"] > max_lat else max_lat
        max_lon = point["lon"] if point["lon"] > max_lon else max_lon

    return {
        "min_lat": min_lat,
        "min_lon": min_lon,
        "max_lat": max_lat,
        "max_lon": max_lon
    }

def points_to_coordinates(points, bounds, desired_width=1000):
    """ Transforms GPS points into coordinates for the desired SVG file """
    height = bounds["max_lat"] - bounds["min_lat"]
    width = bounds["max_lon"] - bounds["min_lon"]

    transform_coefficient = float(desired_width) / width # Make width 1000

    return map(lambda x: point_to_coordinate(x, bounds, transform_coefficient), points)

def get_n_colors(n):
    """ Returns N contrasting RGB fill descriptors """
    transform_constant = 1.0 / n

    rgb_colors = []

    for i in range(0, n):
        hue = i * transform_constant
        lightness = 0.5 + 0.1 * random.random()
        saturation = 0.9 + 0.1 * random.random()
        rgb = map(lambda x: x * 256.0, colorsys.hls_to_rgb(hue, lightness, saturation))
        rgb_colors.append(svgwrite.rgb(rgb[0], rgb[1], rgb[2], '%'))

    return rgb_colors

def point_to_coordinate(point, bounds, transform_coefficient):
    """ Transforms a single point into a coordinate in an SVG file """
    return ((point["lon"] - bounds["min_lon"]) * transform_coefficient,
            ((bounds["max_lat"] - bounds["min_lat"]) - (point["lat"] - bounds["min_lat"])) * transform_coefficient)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Turns a gpx file into an art.')
    parser.add_argument("-o", "--output", type=argparse.FileType('w'), required=True, help="Directs the output to a name of your choice")
    parser.add_argument('inputs', type=argparse.FileType('r'), nargs='+')

    args = parser.parse_args()
    main(args.inputs, args.output)<commit_msg>Changed tool so that it generates one color per directory full of GPX files<commit_after>from bs4 import BeautifulSoup
import argparse
import svgwrite
import os
import random
import colorsys

def main(input, output):
    input_map = get_input_map(input)

    dwg = svgwrite.Drawing(output.name, profile='tiny')

    colors = get_n_colors(len(input_map))
    color_idx = 0

    parsed_input_map = parse_input_map(input_map)
    bounds = get_universal_bounds(parsed_input_map)

    for input in parsed_input_map:
        coords = points_to_coordinates(parsed_input_map[input]["points"], bounds)
        dwg.add(dwg.polyline(coords, stroke=colors[color_idx], stroke_width="3", stroke_opacity="0.6", fill="none"))
        color_idx += 1

    dwg.save()

def get_input_map(input_dir):
    input_map = {}

    for dir in os.listdir(input_dir):
        dir_path = os.path.join(input_dir,dir)
        input_map[dir] = map(lambda x: open(os.path.join(dir_path, x), 'r'), os.listdir(dir_path))

    return input_map

def parse_input_map(input_map):
    parsed_input_map = {}
    for input in input_map:
        points = []
        for file in input_map[input]:
            points.extend(get_points_from_file(file))
        bounds = get_bounds(points)
        parsed_input_map[input] = {"points": points, "bounds": bounds}
    return parsed_input_map

def get_points_from_file(file):
    """ Parses a GPX file, and returns an array of all GPS points described by that file """
    parsed = BeautifulSoup(file.read(), 'html.parser')
    segments = parsed.trk.findAll("trkseg")
    points = []

    for segment in segments:
        segment_points = segment.findAll("trkpt")
        points.extend(map(lambda x: transform_bs4_point(x), segment_points))

    return points

def transform_bs4_point(bs4_point):
    """ Transforms a single BeautifulSoup node representing a node in the GPS coordinate graph
    into a dictionary representing the elements we care about """
    return {
        "lat": float(bs4_point["lat"]), # Degrees
        "lon": float(bs4_point["lon"]), # Degrees
        "ele": float(bs4_point.ele.string), # Meters?
        "time": bs4_point.time.string # ISO8601
    }

def get_universal_bounds(parsed_input_map):
    """ Gets the bounds of all recorded segments """
    min_lat = float("inf")
    max_lat = float("-inf")
    min_lon = float("inf")
    max_lon = float("-inf")

    for parsed_input in parsed_input_map:
        item = parsed_input_map[parsed_input]

        min_lat = item["bounds"]["min_lat"] if item["bounds"]["min_lat"] < min_lat else min_lat
        min_lon = item["bounds"]["min_lon"] if item["bounds"]["min_lon"] < min_lon else min_lon
        max_lat = item["bounds"]["max_lat"] if item["bounds"]["max_lat"] > max_lat else max_lat
        max_lon = item["bounds"]["max_lon"] if item["bounds"]["max_lon"] > max_lon else max_lon

    return {
        "min_lat": min_lat,
        "min_lon": min_lon,
        "max_lat": max_lat,
        "max_lon": max_lon
    }

def get_bounds(points):
    """ Returns the min and max lat and lon within an array of points """
    min_lat = float("inf")
    max_lat = float("-inf")
    min_lon = float("inf")
    max_lon = float("-inf")

    for point in points:
        min_lat = point["lat"] if point["lat"] < min_lat else min_lat
        min_lon = point["lon"] if point["lon"] < min_lon else min_lon
        max_lat = point["lat"] if point["lat"] > max_lat else max_lat
        max_lon = point["lon"] if point["lon"] > max_lon else max_lon

    return {
        "min_lat": min_lat,
        "min_lon": min_lon,
        "max_lat": max_lat,
        "max_lon": max_lon
    }

def points_to_coordinates(points, bounds, desired_width=1000):
    """ Transforms GPS points into coordinates for the desired SVG file """
    height = bounds["max_lat"] - bounds["min_lat"]
    width = bounds["max_lon"] - bounds["min_lon"]

    transform_coefficient = float(desired_width) / width # Make width 1000

    return map(lambda x: point_to_coordinate(x, bounds, transform_coefficient), points)

def get_n_colors(n):
    """ Returns N contrasting RGB fill descriptors """
    transform_constant = 1.0 / n

    rgb_colors = []

    for i in range(0, n):
        hue = i * transform_constant
        lightness = 0.5 + 0.1 * random.random()
        saturation = 0.9 + 0.1 * random.random()
        rgb = map(lambda x: x * 256.0, colorsys.hls_to_rgb(hue, lightness, saturation))
        rgb_colors.append(svgwrite.rgb(rgb[0], rgb[1], rgb[2], '%'))

    return rgb_colors

def point_to_coordinate(point, bounds, transform_coefficient):
    """ Transforms a single point into a coordinate in an SVG file """
    return ((point["lon"] - bounds["min_lon"]) * transform_coefficient,
            ((bounds["max_lat"] - bounds["min_lat"]) - (point["lat"] - bounds["min_lat"])) * transform_coefficient)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Turns a gpx file into an art.')
    parser.add_argument("-o", "--output", type=argparse.FileType('w'), required=True, help="Directs the output to a name of your choice")
    parser.add_argument("-i", "--input", required=True, help="Specifies the input directory")

    args = parser.parse_args()
    main(args.input, args.output)<|endoftext|>


<commit_before>"""Minimal Ordinary Least Squares Examples
"""

import numpy as np
import statsmodels.api as sm

nsample = 100
x = np.linspace(0,10, 100)
X = sm.add_constant(np.column_stack((x, x**2)))
beta = np.array([1, 0.1, 10])
y = np.dot(X, beta) + np.random.normal(size=nsample)

results = sm.OLS(y, X).fit()
print results.summary()

<commit_msg>More verbose example_ols_minimal.py<commit_after>"""Minimal Ordinary Least Squares Examples
"""

import numpy as np
import statsmodels.api as sm

# Artificial data: :math:`y_i = 1 + .1 * x_i + 1 * x_i^2 + e_i`, with :math:`e\sim N(0,1)`
nsample = 100
x = np.linspace(0, 10, 100)
X = sm.add_constant(np.column_stack((x, x**2)))
beta = np.array([1, 0.1, 10])
e = np.random.normal(size=nsample)
y = np.dot(X, beta) + e

# Inspect data
print X[:5,:]
print y[:5]

# Describe, fit, and summarize OLS model
model = sm.OLS(y, X)
results = model.fit()
print results.summary()

<|endoftext|>


<commit_before>"""Copyright (c) 2010-2012 David Rio Vierra

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE."""

import sys
import os
import json
import glob
import shutil

def win32_utf8_argv():
    """Uses shell32.GetCommandLineArgvW to get sys.argv as a list of UTF-8
    strings.

    Versions 2.5 and older of Python don't support Unicode in sys.argv on
    Windows, with the underlying Windows API instead replacing multi-byte
    characters with '?'.

    Returns None on failure.

    Example usage:

    >>> def main(argv=None):
    ...    if argv is None:
    ...        argv = win32_utf8_argv() or sys.argv
    ...
    """

    try:
        from ctypes import POINTER, byref, cdll, c_int, windll
        from ctypes.wintypes import LPCWSTR, LPWSTR

        GetCommandLineW = cdll.kernel32.GetCommandLineW
        GetCommandLineW.argtypes = []
        GetCommandLineW.restype = LPCWSTR

        CommandLineToArgvW = windll.shell32.CommandLineToArgvW
        CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(c_int)]
        CommandLineToArgvW.restype = POINTER(LPWSTR)

        cmd = GetCommandLineW()
        argc = c_int(0)
        argv = CommandLineToArgvW(cmd, byref(argc))
        if argc.value > 0:
            # # Remove Python executable if present
            #            if argc.value - len(sys.argv) == 1:
            #                start = 1
            #            else:
            #                start = 0
            return [argv[i] for i in
                    xrange(0, argc.value)]
    except Exception:
        pass

def getDataDir():
    """Returns the folder where the executable is located."""
    if sys.platform == "win32":
        def fsdecode(x):
            return x.decode(sys.getfilesystemencoding())

        argzero = fsdecode(os.path.abspath(sys.argv[0]))
        if hasattr(sys, 'frozen'):
            dataDir = os.path.dirname(fsdecode(sys.executable))
        else:
            dataDir = os.path.dirname(argzero)
        dataDir = os.getcwdu()
    else:
        dataDir = os.getcwdu()

    return dataDir

def win32_appdata():
    # try to use win32 api to get the AppData folder since python doesn't populate os.environ with unicode strings.

    try:
        import win32com.client

        objShell = win32com.client.Dispatch("WScript.Shell")
        return objShell.SpecialFolders("AppData")
    except Exception, e:
        print "Error while getting AppData folder using WScript.Shell.SpecialFolders: {0!r}".format(e)
        try:
            from win32com.shell import shell, shellcon

            return shell.SHGetPathFromIDListEx(
                shell.SHGetSpecialFolderLocation(0, shellcon.CSIDL_APPDATA)
            )
        except Exception, e:
            print "Error while getting AppData folder using SHGetSpecialFolderLocation: {0!r}".format(e)

            return os.environ['APPDATA'].decode(sys.getfilesystemencoding())

def getCacheDir():
    """Returns the path to the cache folder. This folder is the Application Support folder on OS X, and the Documents Folder on Windows."""
    if sys.platform == "win32":
        return os.path.join(os.path.join(getDocumentsFolder(),'MCEdit'))
    elif sys.platform == "darwin":
        return os.path.expanduser("~/Library/Application Support/pymclevel")
    else:
        return os.path.expanduser("~/.pymclevel")

def getMinecraftProfileJSON():
    """Returns a dictionary object with the minecraft profile information"""
    if os.path.isfile(os.path.join(getMinecraftLauncherDirectory(), "launcher_profiles.json")):
        try:
            with open(os.path.join(getMinecraftLauncherDirectory(), "launcher_profiles.json")) as jsonString:
                minecraftProfilesJSON = json.load(jsonString)
            return minecraftProfilesJSON
        except:
            return None

def getMinecraftProfileDirectory(profileName):
    """Returns the path to the sent minecraft profile directory"""
    try:
        profileDir = getMinecraftProfileJSON()['profiles'][profileName][
            'gameDir']  # profileDir update to correct location.
        return profileDir
    except:
        return os.path.join(getMinecraftLauncherDirectory())

def getMinecraftLauncherDirectory():
    """Returns the /minecraft directory, note: may not contain the /saves folder!"""
    if sys.platform == "win32":
        return os.path.join(win32_appdata(), ".minecraft")
    elif sys.platform == "darwin":
        return os.path.expanduser("~/Library/Application Support/minecraft")
    else:
        return os.path.expanduser("~/.minecraft")

def getDocumentsFolder():
    docsFolder = None

    if sys.platform == "win32":
        try:
            import win32com.client
            objShell = win32com.client.Dispatch("WScript.Shell")
            docsFolder = objShell.SpecialFolders("MyDocuments")

        except Exception, e:
            print e
            try:
                docsFolder = shell.SHGetFolderPath(0, shellcon.CSIDL_MYDOCUMENTS, 0, 0)
            except Exception, e:
                userprofile = os.environ['USERPROFILE'].decode(sys.getfilesystemencoding())
                docsFolder = os.path.join(userprofile, "Documents")

    elif sys.platform == "darwin":
        docsFolder = os.path.expanduser("~/Documents")
    else:
        docsFolder = os.path.expanduser("~/.mcedit")
    try:
        os.mkdir(docsFolder)
    except:
        pass

    return docsFolder

def getSelectedProfile():
    try:
        selectedProfile = getMinecraftProfileJSON()['selectedProfile']
        return selectedProfile
    except:
        return None

def getAllFilters(filters_dir):
    return glob.glob(filters_dir+"/*.py")

# Create pymclevel folder as needed
if not os.path.exists(getCacheDir()):
    os.makedirs(getCacheDir())

# set userCachePath
userCachePath = os.path.join(getCacheDir(),'usercache.json')
# Make sure it exists
try:
    if not os.path.exists(userCachePath):
        f = open(userCachePath,'w')
        f.write('{}')
        f.close()
except:
    print "Unable to make usercache.json at {}".format(userCachePath)

minecraftSaveFileDir = os.path.join(getMinecraftProfileDirectory(getSelectedProfile()), "saves")

ini = u"mcedit.ini"
cache = u"usercache.json"

parentDir = os.path.dirname(getDataDir())
docsFolder = os.path.join(getDocumentsFolder(),'MCEdit')

portableConfigFilePath = os.path.join(parentDir, cache)
portableCacheFilePath = os.path.join(parentDir, ini)
portableSchematicsDir = os.path.join(parentDir, u"Schematics")
portableJarStorageDir = os.path.join(parentDir, u"ServerJarStorage")
portableFiltersDir = os.path.join(parentDir, u"Filters")
if not os.path.exists(parentDir):
    os.makedirs(parentDir)

fixedCacheFilePath = os.path.join(docsFolder, cache)
fixedConfigFilePath = os.path.join(docsFolder, ini)
fixedSchematicsDir = os.path.join(docsFolder, u"Schematics")
FixedJarStorageDir = os.path.join(docsFolder, u"ServerJarStorage")
fixedFiltersDir = os.path.join(docsFolder, u"Filters")
if not os.path.exists(docsFolder):
    os.makedirs(docsFolder)

if sys.platform == "darwin":
    # parentDir is MCEdit.app/Contents/
    if ".app" in parentDir:
        folderContainingAppPackage = os.path.dirname(os.path.dirname(parentDir)) # Running frmo app bundle
    else:
        folderContainingAppPackage = parentDir # Running from source
    oldPath = fixedConfigFilePath

    fixedConfigFilePath = os.path.expanduser("~/Library/Preferences/mcedit.ini")
    fixedSchematicsDir = os.path.join(getCacheDir(), u"Schematics")
    fixedFiltersDir = os.path.join(getCacheDir(), u"Filters")
    if not os.path.exists(getCacheDir()):
        os.makedirs(getCacheDir())

    if os.path.exists(oldPath):
        try:
            os.rename(oldPath, fixedConfigFilePath)
        except Exception, e:
            print repr(e)

    portableConfigFilePath = os.path.join(folderContainingAppPackage, ini)
    portableSchematicsDir = os.path.join(folderContainingAppPackage, u"MCEdit/Schematics")
    portableFiltersDir = os.path.join(folderContainingAppPackage, u"MCEdit/Filters")
    try:
        if not os.path.exists(os.path.join(folderContainingAppPackage,"MCEdit")):
            os.makedirs(os.path.join(folderContainingAppPackage,"MCEdit"))
    except:
        print "Error making {}".format(os.path.join(folderContainingAppPackage,"MCEdit"))

print portableFiltersDir

def goPortable():
    global configFilePath, schematicsDir, filtersDir, portable

    if os.path.exists(fixedSchematicsDir):
        move_displace(fixedSchematicsDir, portableSchematicsDir)
    if os.path.exists(fixedConfigFilePath):
        move_displace(fixedConfigFilePath, portableConfigFilePath)
    if os.path.exists(fixedFiltersDir):
        move_displace(fixedFiltersDir, portableFiltersDir)

    schematicsDir = portableSchematicsDir
    configFilePath = portableConfigFilePath
    filtersDir = portableFiltersDir
    portable = True

def move_displace(src, dst):
    dstFolder = os.path.basename(os.path.dirname(dst))
    if not os.path.exists(dst):

        print "Moving {0} to {1}".format(os.path.basename(src), dstFolder)
        shutil.move(src, dst)
    else:
        olddst = dst + ".old"
        i = 0
        while os.path.exists(olddst):
            olddst = dst + ".old" + str(i)
            i += 1

        print "{0} already found in {1}! Renamed it to {2}.".format(os.path.basename(src), dstFolder, dst)
        os.rename(dst, olddst)
        shutil.move(src, dst)

def goFixed():
    global configFilePath, schematicsDir, filtersDir, portable

    if os.path.exists(portableSchematicsDir):
        move_displace(portableSchematicsDir, fixedSchematicsDir)
    if os.path.exists(portableConfigFilePath):
        move_displace(portableConfigFilePath, fixedConfigFilePath)
    if os.path.exists(portableFiltersDir):
        move_displace(portableFiltersDir, fixedFiltersDir)

    schematicsDir = fixedSchematicsDir
    configFilePath = fixedConfigFilePath
    filtersDir = fixedFiltersDir
    portable = False

def portableConfigExists():
    return (os.path.exists(portableConfigFilePath)  # mcedit.ini in MCEdit folder
            or (sys.platform != 'darwin' and not os.path.exists(
        fixedConfigFilePath)))  # no mcedit.ini in Documents folder (except on OS X when we always want it in Library/Preferences

if portableConfigExists():
    print "Running in portable mode. MCEdit/Schematics, MCEdit/Filters, and mcedit.ini are stored alongside " + (
    sys.platform == "darwin" and "MCEdit.app" or "MCEditData")
    portable = True
    schematicsDir = portableSchematicsDir
    configFilePath = portableConfigFilePath
    filtersDir = portableFiltersDir

else:
    print "Running in fixed install mode. MCEdit/Schematics, MCEdit/Filters, and mcedit.ini are in your " + (
    sys.platform == "darwin" and "App Support Folder (Available from the main menu of MCEdit)" or "Documents folder.")
    schematicsDir = fixedSchematicsDir
    configFilePath = fixedConfigFilePath
    filtersDir = fixedFiltersDir
    portable = False

#if portable:
#    serverJarStorageDir = (os.path.join(parentDir, "ServerJarStorage"))
#    ServerJarStorage.defaultCacheDir = serverJarStorageDir
#    jarStorage = ServerJarStorage(serverJarStorageDir)
#else:
#    jarStorage = ServerJarStorage()

<commit_msg>Change to directories on Windows<commit_after>"""Copyright (c) 2010-2012 David Rio Vierra

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE."""

import sys
import os
import json
import glob
import shutil

def win32_utf8_argv():
    """Uses shell32.GetCommandLineArgvW to get sys.argv as a list of UTF-8
    strings.

    Versions 2.5 and older of Python don't support Unicode in sys.argv on
    Windows, with the underlying Windows API instead replacing multi-byte
    characters with '?'.

    Returns None on failure.

    Example usage:

    >>> def main(argv=None):
    ...    if argv is None:
    ...        argv = win32_utf8_argv() or sys.argv
    ...
    """

    try:
        from ctypes import POINTER, byref, cdll, c_int, windll
        from ctypes.wintypes import LPCWSTR, LPWSTR

        GetCommandLineW = cdll.kernel32.GetCommandLineW
        GetCommandLineW.argtypes = []
        GetCommandLineW.restype = LPCWSTR

        CommandLineToArgvW = windll.shell32.CommandLineToArgvW
        CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(c_int)]
        CommandLineToArgvW.restype = POINTER(LPWSTR)

        cmd = GetCommandLineW()
        argc = c_int(0)
        argv = CommandLineToArgvW(cmd, byref(argc))
        if argc.value > 0:
            # # Remove Python executable if present
            #            if argc.value - len(sys.argv) == 1:
            #                start = 1
            #            else:
            #                start = 0
            return [argv[i] for i in
                    xrange(0, argc.value)]
    except Exception:
        pass

def getDataDir():
    """Returns the folder where the executable is located."""
    if sys.platform == "win32":
        def fsdecode(x):
            return x.decode(sys.getfilesystemencoding())

        argzero = fsdecode(os.path.abspath(sys.argv[0]))
        if hasattr(sys, 'frozen'):
            dataDir = os.path.dirname(fsdecode(sys.executable))
        else:
            dataDir = os.path.dirname(argzero)
        dataDir = os.getcwdu()
    else:
        dataDir = os.getcwdu()

    return dataDir

def win32_appdata():
    # try to use win32 api to get the AppData folder since python doesn't populate os.environ with unicode strings.

    try:
        import win32com.client

        objShell = win32com.client.Dispatch("WScript.Shell")
        return objShell.SpecialFolders("AppData")
    except Exception, e:
        print "Error while getting AppData folder using WScript.Shell.SpecialFolders: {0!r}".format(e)
        try:
            from win32com.shell import shell, shellcon

            return shell.SHGetPathFromIDListEx(
                shell.SHGetSpecialFolderLocation(0, shellcon.CSIDL_APPDATA)
            )
        except Exception, e:
            print "Error while getting AppData folder using SHGetSpecialFolderLocation: {0!r}".format(e)

            return os.environ['APPDATA'].decode(sys.getfilesystemencoding())

def getCacheDir():
    """Returns the path to the cache folder. This folder is the Application Support folder on OS X, and the Documents Folder on Windows."""
    if sys.platform == "win32":
        return os.path.join(os.path.join(getDocumentsFolder(),'MCEdit'))
    elif sys.platform == "darwin":
        return os.path.expanduser("~/Library/Application Support/pymclevel")
    else:
        return os.path.expanduser("~/.pymclevel")

def getMinecraftProfileJSON():
    """Returns a dictionary object with the minecraft profile information"""
    if os.path.isfile(os.path.join(getMinecraftLauncherDirectory(), "launcher_profiles.json")):
        try:
            with open(os.path.join(getMinecraftLauncherDirectory(), "launcher_profiles.json")) as jsonString:
                minecraftProfilesJSON = json.load(jsonString)
            return minecraftProfilesJSON
        except:
            return None

def getMinecraftProfileDirectory(profileName):
    """Returns the path to the sent minecraft profile directory"""
    try:
        profileDir = getMinecraftProfileJSON()['profiles'][profileName][
            'gameDir']  # profileDir update to correct location.
        return profileDir
    except:
        return os.path.join(getMinecraftLauncherDirectory())

def getMinecraftLauncherDirectory():
    """Returns the /minecraft directory, note: may not contain the /saves folder!"""
    if sys.platform == "win32":
        return os.path.join(win32_appdata(), ".minecraft")
    elif sys.platform == "darwin":
        return os.path.expanduser("~/Library/Application Support/minecraft")
    else:
        return os.path.expanduser("~/.minecraft")

def getDocumentsFolder():
    docsFolder = None

    if sys.platform == "win32":
        try:
            import win32com.client
            objShell = win32com.client.Dispatch("WScript.Shell")
            docsFolder = objShell.SpecialFolders("MyDocuments")

        except Exception, e:
            print e
            try:
                docsFolder = shell.SHGetFolderPath(0, shellcon.CSIDL_MYDOCUMENTS, 0, 0)
            except Exception, e:
                userprofile = os.environ['USERPROFILE'].decode(sys.getfilesystemencoding())
                docsFolder = os.path.join(userprofile, "Documents")

    elif sys.platform == "darwin":
        docsFolder = os.path.expanduser("~/Documents")
    else:
        docsFolder = os.path.expanduser("~/.mcedit")
    try:
        os.mkdir(docsFolder)
    except:
        pass

    return docsFolder

def getSelectedProfile():
    try:
        selectedProfile = getMinecraftProfileJSON()['selectedProfile']
        return selectedProfile
    except:
        return None

def getAllFilters(filters_dir):
    return glob.glob(filters_dir+"/*.py")

# Create pymclevel folder as needed
if not os.path.exists(getCacheDir()):
    os.makedirs(getCacheDir())

# set userCachePath
userCachePath = os.path.join(getCacheDir(),'usercache.json')
# Make sure it exists
try:
    if not os.path.exists(userCachePath):
        f = open(userCachePath,'w')
        f.write('{}')
        f.close()
except:
    print "Unable to make usercache.json at {}".format(userCachePath)

minecraftSaveFileDir = os.path.join(getMinecraftProfileDirectory(getSelectedProfile()), "saves")

ini = u"mcedit.ini"
cache = u"usercache.json"

parentDir = os.path.dirname(getDataDir())
docsFolder = os.path.join(getDocumentsFolder(),'MCEdit')

portableConfigFilePath = os.path.join(parentDir, cache)
portableCacheFilePath = os.path.join(parentDir, ini)
portableSchematicsDir = os.path.join(parentDir, u"MCEdit/Schematics")
portableJarStorageDir = os.path.join(parentDir, u"MCEdit/ServerJarStorage")
portableFiltersDir = os.path.join(parentDir, u"MCEdit/Filters")
if not os.path.exists(os.path.join(parendetDir,"MCEdit")):
    os.makedirs(os.path.join(parendetDir,"MCEdit"))

fixedCacheFilePath = os.path.join(docsFolder, cache)
fixedConfigFilePath = os.path.join(docsFolder, ini)
fixedSchematicsDir = os.path.join(docsFolder, u"MCEdit/Schematics")
FixedJarStorageDir = os.path.join(docsFolder, u"MCEdit/ServerJarStorage")
fixedFiltersDir = os.path.join(docsFolder, u"MCEdit/Filters")
if not os.path.exists(os.path.join(docsFolder,"MCEdit")):
    os.makedirs(os.path.join(docsFolder,"MCEdit"))

if sys.platform == "darwin":
    # parentDir is MCEdit.app/Contents/
    if ".app" in parentDir:
        folderContainingAppPackage = os.path.dirname(os.path.dirname(parentDir)) # Running frmo app bundle
    else:
        folderContainingAppPackage = parentDir # Running from source
    oldPath = fixedConfigFilePath

    fixedConfigFilePath = os.path.expanduser("~/Library/Preferences/mcedit.ini")
    fixedSchematicsDir = os.path.join(getCacheDir(), u"Schematics")
    fixedFiltersDir = os.path.join(getCacheDir(), u"Filters")
    if not os.path.exists(getCacheDir()):
        os.makedirs(getCacheDir())

    if os.path.exists(oldPath):
        try:
            os.rename(oldPath, fixedConfigFilePath)
        except Exception, e:
            print repr(e)

    portableConfigFilePath = os.path.join(folderContainingAppPackage, ini)
    portableSchematicsDir = os.path.join(folderContainingAppPackage, u"MCEdit/Schematics")
    portableFiltersDir = os.path.join(folderContainingAppPackage, u"MCEdit/Filters")
    try:
        if not os.path.exists(os.path.join(folderContainingAppPackage,"MCEdit")):
            os.makedirs(os.path.join(folderContainingAppPackage,"MCEdit"))
    except:
        print "Error making {}".format(os.path.join(folderContainingAppPackage,"MCEdit"))

print portableFiltersDir

def goPortable():
    global configFilePath, schematicsDir, filtersDir, portable

    if os.path.exists(fixedSchematicsDir):
        move_displace(fixedSchematicsDir, portableSchematicsDir)
    if os.path.exists(fixedConfigFilePath):
        move_displace(fixedConfigFilePath, portableConfigFilePath)
    if os.path.exists(fixedFiltersDir):
        move_displace(fixedFiltersDir, portableFiltersDir)

    schematicsDir = portableSchematicsDir
    configFilePath = portableConfigFilePath
    filtersDir = portableFiltersDir
    portable = True

def move_displace(src, dst):
    dstFolder = os.path.basename(os.path.dirname(dst))
    if not os.path.exists(dst):

        print "Moving {0} to {1}".format(os.path.basename(src), dstFolder)
        shutil.move(src, dst)
    else:
        olddst = dst + ".old"
        i = 0
        while os.path.exists(olddst):
            olddst = dst + ".old" + str(i)
            i += 1

        print "{0} already found in {1}! Renamed it to {2}.".format(os.path.basename(src), dstFolder, dst)
        os.rename(dst, olddst)
        shutil.move(src, dst)

def goFixed():
    global configFilePath, schematicsDir, filtersDir, portable

    if os.path.exists(portableSchematicsDir):
        move_displace(portableSchematicsDir, fixedSchematicsDir)
    if os.path.exists(portableConfigFilePath):
        move_displace(portableConfigFilePath, fixedConfigFilePath)
    if os.path.exists(portableFiltersDir):
        move_displace(portableFiltersDir, fixedFiltersDir)

    schematicsDir = fixedSchematicsDir
    configFilePath = fixedConfigFilePath
    filtersDir = fixedFiltersDir
    portable = False

def portableConfigExists():
    return (os.path.exists(portableConfigFilePath)  # mcedit.ini in MCEdit folder
            or (sys.platform != 'darwin' and not os.path.exists(
        fixedConfigFilePath)))  # no mcedit.ini in Documents folder (except on OS X when we always want it in Library/Preferences

if portableConfigExists():
    print "Running in portable mode. MCEdit/Schematics, MCEdit/Filters, and mcedit.ini are stored alongside " + (
    sys.platform == "darwin" and "MCEdit.app" or "MCEditData")
    portable = True
    schematicsDir = portableSchematicsDir
    configFilePath = portableConfigFilePath
    filtersDir = portableFiltersDir

else:
    print "Running in fixed install mode. MCEdit/Schematics, MCEdit/Filters, and mcedit.ini are in your " + (
    sys.platform == "darwin" and "App Support Folder (Available from the main menu of MCEdit)" or "Documents folder.")
    schematicsDir = fixedSchematicsDir
    configFilePath = fixedConfigFilePath
    filtersDir = fixedFiltersDir
    portable = False

#if portable:
#    serverJarStorageDir = (os.path.join(parentDir, "ServerJarStorage"))
#    ServerJarStorage.defaultCacheDir = serverJarStorageDir
#    jarStorage = ServerJarStorage(serverJarStorageDir)
#else:
#    jarStorage = ServerJarStorage()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Cube File Parsing and Composing
=============================================

'''
from atomic import _pd as pd
from atomic.algorithms.nonjitted import generate_minimal_framedf_from_onedf as _gen_fdf

def read_cubes(paths, frames=None, volidxs=None, labels=None,
               universe=None, metadata={}, **kwargs):
    '''
    Args
        paths (list): List of paths to cube files
        frames (list or int): List of same length as paths or frame of interest
        volidxs (list or int): List of same length as paths or starting volidx
        labels (list): List of same length as paths
        universe (:class:`~atomic.container.Universe`): universe to which field belongs (see Note)

    Returns
        universe: New universe or modified universe

    Note
        See parse_cube() for the singular case.
        In order for the cube files to be attached correctly to a universe, all
        indices (frame, volidx) must be specified and the original universe
        must be provided. Otherwise it will create a universe with the given information.

    Warning
        If attaching to an existing universe only the cubedata and cube tables will
        be updated! The frame and one body tables will not be modified!
    '''
    if type(paths) != list:
        paths = [paths]
    vollist = []
    voldatlist = []
    unikws = {}
    cfidx = None
    if frames is None:
        if universe is None:
            frame = 0
        else:
            frame = universe.framelist[0]
    elif frames is int:
        frame = frames
    elif frames is list:
        cfidx = True
    cvidx = None
    if volidxs is None:
        if universe is None:
            volidx = 0
        else:
            volidx = universe.voldat.loc[frame].index.get_level_values('volidx')[-1] + 1
    elif volidxs is int:
        volidx = volidxs
    elif volidxs is list:
        cvidx = True
    clidx = None
    if labels:
        clidx = True
    else:
        label = None

    for i, fl in enumerate(paths):
        df = pd.read_csv(fl, delim_whitespace=True, header=None,
                         skiprows=[0, 1], names=range(6), dtype=float)

        nat = int(df.iloc[0, 0])   # Always needed

        if cfidx:
            frame = frames[i]
        volidx += 1
        if cvidx:
            volidx = volidxs[i]
        if clidx:
            label = labels[i]

        index = (frame, volidx)    # Generate voldata
        voldat, convert_xyz = _gen_voldat(df.iloc[0:4],frame, volidx, label)

                                   # Generate the cube entry
        volume = _gen_volume(df.iloc[nat + 4:], frame, volidx)
        vollist.append(volume)
        voldatlist.append(voldat)

        if i == 0:                 # Only need one of these dfs each
            unikws['one'] = _gen_one(df.iloc[4:nat + 4], convert_xyz, frame)
            unikws['frame'] = _gen_fdf(unikws['one'])

    if universe is None:
        unikws['voldat'] = pd.concat(voldatlist)
        unikws['volume'] = pd.concat(vollist)
        unikws['metadata'] = {'paths': paths}
        unikws.update(kwargs)
        #uni = Universe(**unikws)
        #uni.get_two_body_properties()
        return unikws
    else:
        o = ['frame', 'volidx']
        if universe.volume is None:
            universe.volume = volume
        else:
            print('If cubes exist for add cubes this fails currently.')
            pass
            #if index in universe.volume.index:
            #    universe.volume.set_value(index, 'mag', volume.values)
#            else:
#                universe.volume = universe.volume.append(volume).reset_index(o).sort_values(o).set_index(o)
#        if universe.voldat is None:
#            universe.voldat = voldat
#        else:
#            mymap = voldat.to_dict()
#            if index in universe.voldat.index:
#                universe.voldat.iloc[index].map(mymap)
#            else:
#                universe.voldat = universe.voldat.append(voldat).reset_index(o).sort_values(o).set_index(o)
        return universe

#def read_cube(cube_file, frame=None, volidx=None, label=None, spin=None, universe=None, **kwargs):
#    '''
#    Args
#        cube_file (str): Cube file path
#        frame (int): Frame index (frame) to which this field belongs (see Note)
#        volidx (int): Volume index (may repeat volume additions if not specified)
#        label (str): String description of the volume in the cube file
#        spin (int): If an orbital cube, may optionally specify the spin
#        universe (:class:`~exa.atomic.container.Universe`): Universe where the field belongs (see Note)
#
#    Returns
#        universe: New universe or modified original universe
#
#    Note
#        In order for the cube file to be attached correctly to a universe, all
#        indices (frame, volidx) must be specified as well as the original universe
#        must be provided. Otherwise it will create a universe with the given information.
#
#    Warning
#        If attaching to an existing universe only the cubedata and cube tables will
#        be updated! The frame and one body tables will not be modified!
#    '''
#    df = pd.read_csv(cube_file, delim_whitespace=True, header=None,
#                     skiprows=[0, 1], names=range(6), dtype=float)
#
#    # Always needed
#    nat = int(df.iloc[0, 0])
#    if frame is None:
#        frame = 0 if universe is None else universe.framelist[0]
#    if volidx is None:
#        volidx = 0 if universe is None else universe.voldat.loc[frame].index.get_level_values('volidx')[-1] + 1
#
#    # Generate voldata
#    index = (frame, volidx)
#    voldat, convert_xyz = _gen_voldat(df.iloc[0:4],frame, volidx, label, spin)
#
#    # Generate the cube entry
#    volume = _gen_volume(df.iloc[nat + 4:], frame, volidx)
#
#    if universe is None:
#        unikws = {}
#        # Generate the frame table
#        unikwargs['frame'] = _gen_fdf(nat, frame)
#        # Generate the one body table
#        unikwargs['one'] = _gen_one(df.iloc[4:nat + 4], convert_xyz, frame)
#        # Attach
#        unikwargs['voldat'] = voldat
#        unikwargs['volume'] = volume
#        # In case the user supplied project or job information etc.
#        unikwargs.update(kwargs)
#        uni = Universe(**unikwargs)
#        uni.get_two_body_properties()
#        return uni
#    else:
#        o = ['frame', 'volidx']
#        if universe.volume is None:
#            universe.volume = volume
#        else:
#            if index in universe.volume.index:
#                universe.volume.set_value(index, 'mag', volume.values)
#            else:
#                universe.volume = universe.volume.append(volume).reset_index(o).sort_values(o).set_index(o)
#        if universe.voldat is None:
#            universe.voldat = voldat
#        else:
#            mymap = voldat.to_dict()
#            if index in universe.voldat.index:
#                universe.voldat.iloc[index].map(mymap)
#            else:
#                universe.voldat = universe.voldat.append(voldat).reset_index(o).sort_values(o).set_index(o)

def _gen_voldat(data, frame, volidx, label):
    '''
    '''
    convert_xyz = False
    origin = data.iloc[0, 1:4].values
    v = data.iloc[1:].unstack().dropna().values
    voldatdict = {'ox': origin[0], 'oy': origin[1], 'oz': origin[2],
                  'nx': v[0], 'dxi': v[3], 'dxj': v[4], 'dxk': v[5],
                  'ny': v[1], 'dyi': v[6], 'dyj': v[7], 'dyk': v[8],
                  'nz': v[2], 'dzi': v[9], 'dzj': v[10], 'dzk': v[11],
                  'label': label}
    df = pd.DataFrame(voldatdict, index=[0])
    # Check units
    for i in ['x', 'y', 'z']:
        if any(df['n' + i]) < 0:
            convert_xyz = True
            df['n' + i] *= -1
            df['d' + i + 'i'] *= Length['A', 'a0']
            df['d' + i + 'j'] *= Length['A', 'a0']
            df['d' + i + 'k'] *= Length['A', 'a0']
    df['frame'] = frame
    df['volidx'] = volidx
    df.set_index(['frame', 'volidx'], inplace=True)
    return df, convert_xyz

def _gen_one(onedf, convert_xyz, frame):
    '''
    '''
    distinct = onedf[0].astype(int).unique()
    #sym_dict = iso[iso.index.isin(distinct)]['symbol'].to_dict()
    df = onedf.loc[:, [0, 2, 3, 4]].reset_index(drop=True)
    df.index.names = ['one']
    df.columns = ['symbol', 'x', 'y', 'z']
    if convert_xyz:
        df[['x', 'y', 'z']] *= Length['A', 'a0']
    #df['symbol'] = df['symbol'].apply(lambda sym: sym_dict[sym])
    df['frame'] = frame
    df.set_index('frame', append=True, inplace=True)
    df = df.reorder_levels(['frame', 'one'])
    return df

def _gen_volume(data, frame, volidx):
    '''
    '''
    df = data.stack().dropna().reset_index(drop=True).to_frame()
    df.columns = ['mag']
    df.index.names = ['data']
    df['frame'] = frame
    df['volidx'] = volidx
    # Next line of code is the slow step (but not due to inplace=True)
    df.set_index(['frame', 'volidx'], append=True, inplace=True)
    df = df.reorder_levels(['frame', 'volidx', 'data'])
    return df
<commit_msg>Implemented most features of read_cubes, properly accepts frame(s), volidx(s), labels, accepting universe is untested.<commit_after># -*- coding: utf-8 -*-
'''
Cube File Parsing and Composing
=============================================

'''
# Hacky imports
import sys
sys.path.insert(0, '/home/tjd/Programs/analytics-exa/exa')

from exa import Isotope
from atomic import _pd as pd
from atomic.algorithms.nonjitted import generate_minimal_framedf_from_onedf as _gen_fdf

def read_cubes(paths, frames=None, volidxs=None, labels=None,
               universe=None, metadata={}, **kwargs):
    '''
    Args
        paths (str or list): str to path or list of paths to cube files
        frames (int or list): frame of interest or list of same length as paths
        volidxs (int or list): starting volidx or list of same length as paths
        labels (list): List of same length as paths
        universe (:class:`~atomic.container.Universe`): universe to which field belongs (see Note)

    Returns
        unikws: New universe or modified universe

    Note
        See parse_cube() for the singular case.
        In order for the cube files to be attached correctly to a universe, all
        indices (frame, volidx) must be specified and the original universe
        must be provided. Otherwise it will create a universe with the given information.

    Warning
        If attaching to an existing universe only the cubedata and cube tables will
        be updated! The frame and one body tables will not be modified!
    '''
    if type(paths) != list:
        paths = [paths]
    vollist = []
    voldatlist = []
    unikws = {}

    cfidx = None
    if frames is None:
        if universe is None:
            frame = 0
        else:
            frame = universe.framelist[0]
    elif type(frames) is int:
        frame = frames
    elif type(frames) is list:
        cfidx = True
        frame = frames[0]

    cvidx = None
    if volidxs is None:
        if universe is None:
            volidx = 0
        else:
            volidx = universe.voldat.loc[frame].index.get_level_values('volidx')[-1] + 1
    elif type(volidxs) is int:
        volidx = volidxs
    elif type(volidxs) is list:
        cvidx = True
    clidx = None
    if labels:
        clidx = True
        label = labels[0]
    else:
        label = None

    print(frame)
    print(label)
    print(volidx)

    onelist = []
    framelist = []

    for i, fl in enumerate(paths):
        df = pd.read_csv(fl, delim_whitespace=True, header=None,
                         skiprows=[0, 1], names=range(6), dtype=float)

        nat = int(df.iloc[0, 0])   # Always needed

        if cfidx:
            frame = frames[i]
        else:
            volidx += 1
        if cvidx:
            volidx = volidxs[i]
        if clidx:
            label = labels[i]

        index = (frame, volidx)    # Generate voldata
        voldat, convert_xyz = _gen_voldat(df.iloc[0:4],frame, volidx, label)

                                   # Generate the cube entry
        volume = _gen_volume(df.iloc[nat + 4:], frame, volidx)
        vollist.append(volume)
        voldatlist.append(voldat)

        if cfidx:
            onelist.append(_gen_one(df.iloc[4:nat + 4], convert_xyz, frame))
            framelist.append(_gen_fdf(onelist[-1]))
        else:
            if i == 0:             # Only need one of these dfs each
                unikws['one'] = _gen_one(df.iloc[4:nat + 4], convert_xyz, frame)
                unikws['frame'] = _gen_fdf(unikws['one'])

    if universe is None:
        if cfidx:
            unikws['one'] = pd.concat(onelist)
            unikws['frame'] = pd.concat(framelist)
        unikws['voldat'] = pd.concat(voldatlist)
        unikws['volume'] = pd.concat(vollist)
        unikws['metadata'] = {'paths': paths}
        unikws.update(kwargs)
        #uni = Universe(**unikws)
        #uni.get_two_body_properties()
        return unikws
    else:
        o = ['frame', 'volidx']
        if universe.volume is None:
            universe.volume = volume
        else:
            if index in universe.volume.index:
                universe.volume.set_value(index, 'mag', volume.values)
            else:
                universe.volume = universe.volume.append(volume).reset_index(o).sort_values(o).set_index(o)
        if universe.voldat is None:
            universe.voldat = voldat
        else:
            mymap = voldat.to_dict()
            if index in universe.voldat.index:
                universe.voldat.iloc[index].map(mymap)
            else:
                universe.voldat = universe.voldat.append(voldat).reset_index(o).sort_values(o).set_index(o)
        return universe

def _gen_voldat(data, frame, volidx, label):
    '''
    Generates 'voldat' dataframe (volume metadata)
    '''
    convert_xyz = False
    origin = data.iloc[0, 1:4].values
    v = data.iloc[1:].unstack().dropna().values
    voldatdict = {'ox': origin[0], 'oy': origin[1], 'oz': origin[2],
                  'nx': v[0], 'dxi': v[3], 'dxj': v[4], 'dxk': v[5],
                  'ny': v[1], 'dyi': v[6], 'dyj': v[7], 'dyk': v[8],
                  'nz': v[2], 'dzi': v[9], 'dzj': v[10], 'dzk': v[11],
                  'label': label}
    df = pd.DataFrame(voldatdict, index=[0])
    # Check units
    for i in ['x', 'y', 'z']:
        if any(df['n' + i]) < 0:
            convert_xyz = True
            df['n' + i] *= -1
            df['d' + i + 'i'] *= Length['A', 'a0']
            df['d' + i + 'j'] *= Length['A', 'a0']
            df['d' + i + 'k'] *= Length['A', 'a0']
    df['frame'] = frame
    df['volidx'] = volidx
    df.set_index(['frame', 'volidx'], inplace=True)
    return df, convert_xyz

def _gen_one(onedf, convert_xyz, frame):
    '''
    Generates 'one' body dataframe
    '''
    distinct = onedf[0].astype(int).unique()
    nisos = [Isotope.get_element(int(znum), by='znum') for znum in distinct]
    isos = {iso.Z: iso.symbol for iso in nisos}
    df = onedf.loc[:, [0, 2, 3, 4]].reset_index(drop=True)
    df.index.names = ['one']
    df.columns = ['symbol', 'x', 'y', 'z']
    if convert_xyz:
        df[['x', 'y', 'z']] *= Length['A', 'a0']
    df['symbol'] = df['symbol'].map(isos)
    df['frame'] = frame
    df.set_index('frame', append=True, inplace=True)
    df = df.reorder_levels(['frame', 'one'])
    return df

def _gen_volume(data, frame, volidx):
    '''
    Generates 'volume' dataframe
    '''
    df = data.stack().dropna().reset_index(drop=True).to_frame()
    df.columns = ['mag']
    df.index.names = ['data']
    df['frame'] = frame
    df['volidx'] = volidx
    # Next line of code is the slow step (but not due to inplace=True)
    df.set_index(['frame', 'volidx'], append=True, inplace=True)
    df = df.reorder_levels(['frame', 'volidx', 'data'])
    return df
<|endoftext|>