<commit_before>"""
A Spawner for JupyterHub that runs each user's
server in a separate Docker Service
"""

import os
import string
import hashlib
from textwrap import dedent
from concurrent.futures import ThreadPoolExecutor
from pprint import pformat

import docker
from docker.errors import APIError
from docker.utils import kwargs_from_env
from tornado import gen

from jupyterhub.spawner import Spawner
from traitlets import (
    Dict,
    Unicode,
    List,
    Bool,
    Int,
)

class UnicodeOrFalse(Unicode):
    info_text = 'a unicode string or False'

    def validate(self, obj, value):
        if value is False:
            return value
        return super(UnicodeOrFalse, self).validate(obj, value)

class SwarmSpawner(Spawner):
    """A Spawner for JupyterHub using Docker Engine in Swarm mode
    """

    _executor = None

    @property
    def executor(self, max_workers=1):
        """single global executor"""
        cls = self.__class__
        if cls._executor is None:
            cls._executor = ThreadPoolExecutor(max_workers)
        return cls._executor

    _client = None

    @property
    def client(self):
        """single global client instance"""
        cls = self.__class__
        if cls._client is None:
            if self.use_docker_client_env:
                kwargs = kwargs_from_env(
                    assert_hostname=self.tls_assert_hostname
                )
                client = docker.APIClient(version='auto', **kwargs)
            else:
                if self.tls:
                    tls_config = True
                elif self.tls_verify or self.tls_ca or self.tls_client:
                    tls_config = docker.tls.TLSConfig(
                        client_cert=self.tls_client,
                        ca_cert=self.tls_ca,
                        verify=self.tls_verify,
                        assert_hostname=self.tls_assert_hostname)
                else:
                    tls_config = None

                docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')

                client = docker.APIClient(base_url=docker_host,
                                          tls=tls_config,
                                          version='auto')
            cls._client = client
        return cls._client

    service_id = Unicode()
    service_port = Int(8888, min=1, max=65535, config=True)
    service_image = Unicode("jupyterhub/singleuser", config=True)
    service_prefix = Unicode(
        "jupyter",
        config=True,
        help=dedent(
            """
            Prefix for service names. The full service name for a particular
            user will be <prefix>-<hash(username)>-<server_name>.
            """
        )
    )

    use_docker_client_env = Bool(False, config=True,
                                 help="If True, will use Docker client env \
                                 variable (boot2docker friendly)")
    tls = Bool(False, config=True, help="If True, connect to docker with --tls")
    tls_verify = Bool(False, config=True, help="If True, connect to docker with --tlsverify")
    tls_ca = Unicode("", config=True, help="Path to CA certificate for docker TLS")
    tls_cert = Unicode("", config=True, help="Path to client certificate for docker TLS")
    tls_key = Unicode("", config=True, help="Path to client key for docker TLS")
    tls_assert_hostname = UnicodeOrFalse(default_value=None,
                                         allow_none=True,
                                         config=True,
                                         help="If False, do not verify hostname of docker daemon",)

    container_spec = Dict({}, config=True, help="Params to create the service")
    resource_spec = Dict({}, config=True, help="Params about cpu and memory limits")
    networks = List([], config=True, help="Additional args to create_host_config for service create")

    _docker_safe_chars = set(string.ascii_letters + string.digits + '-')
    _docker_escape_char = '_'

    use_user_options = Bool(False, config=True, help="the spawner will use the dict passed through the form or as json body when using the Hub Api")
    jupyterhub_service_name = Unicode(config=True,
                                      help=dedent("""Name of the service running the JupyterHub"""))

    @property
    def tls_client(self):
        """A tuple consisting of the TLS client certificate and key if they
        have been provided, otherwise None.

        """
        if self.tls_cert and self.tls_key:
            return (self.tls_cert, self.tls_key)
        return None

    _service_owner = None

    @property
    def service_owner(self):
        if self._service_owner is None:
            m = hashlib.md5()
            m.update(self.user.name.encode('utf-8'))
            self._service_owner = m.hexdigest()
        return self._service_owner

    @property
    def service_name(self):
        """
        Service name inside the Docker Swarm

        service_suffix should be a numerical value unique for user
        {service_prefix}-{service_owner}-{service_suffix}
        """
        # JupyterHub with multi-server-per-user should pass server_name
        # In the future we can remove this part
        if hasattr(self, 'server_name'):
            server_name = self.server_name
        else:
            server_name = 1

        return "{}-{}-{}".format(self.service_prefix,
                                 self.service_owner,
                                 server_name
                                )

    def load_state(self, state):
        super().load_state(state)
        self.service_id = state.get('service_id', '')

    def get_state(self):
        state = super().get_state()
        if self.service_id:
            state['service_id'] = self.service_id
        return state

    def _env_keep_default(self):
        """it's called in traitlets. It's a special method name.
        Don't inherit any env from the parent process"""
        return []

    def _public_hub_api_url(self):
        proto, path = self.hub.api_url.split('://', 1)
        _, rest = path.split(':', 1)
        return '{proto}://{name}:{rest}'.format(
            proto=proto,
            name=self.jupyterhub_service_name,
            rest=rest
        )

    def get_env(self):
        env = super().get_env()
        env.update(dict(
            JPY_USER=self.user.name,
            JPY_COOKIE_NAME=self.user.server.cookie_name,
            JPY_BASE_URL=self.user.server.base_url,
            JPY_HUB_PREFIX=self.hub.server.base_url
        ))

        if self.notebook_dir:
            env['NOTEBOOK_DIR'] = self.notebook_dir

        env['JPY_HUB_API_URL'] = self._public_hub_api_url()

        return env

    def _docker(self, method, *args, **kwargs):
        """wrapper for calling docker methods

        to be passed to ThreadPoolExecutor
        """
        m = getattr(self.client, method)
        return m(*args, **kwargs)

    def docker(self, method, *args, **kwargs):
        """Call a docker method in a background thread

        returns a Future
        """
        return self.executor.submit(self._docker, method, *args, **kwargs)

    @gen.coroutine
    def poll(self):
        """Check for a task state like `docker service ps id`"""
        service = yield self.get_service()
        if not service:
            self.log.warn("Service not found")
            return ""

        task_filter = {'service': service['Spec']['Name']}

        task = yield self.docker(
            'tasks', task_filter
        )

        # use the first task and only task
        task = task[0]

        task_state = task['Status']['State']
        self.log.debug(
            "Task %s of Service %s status: %s",
            task['ID'][:7],
            self.service_id[:7],
            pformat(task_state),
        )

        if task_state == 'running':
            return None
        else:
            return (
                "ExitCode={ExitCode}, "
                "Error='{Error}', "
                "FinishedAt={FinishedAt}".format(**task_state)
            )

    @gen.coroutine
    def get_service(self):
        self.log.debug("Getting service '%s'", self.service_name)
        try:
            service = yield self.docker(
                'inspect_service', self.service_name
            )
            self.service_id = service['ID']
        except APIError as err:
            if err.response.status_code == 404:
                self.log.info("Service '%s' is gone", self.service_name)
                service = None
                # Docker service is gone, remove service id
                self.service_id = ''
            elif err.response.status_code == 500:
                self.log.info("Docker Swarm Server error")
                service = None
                # Docker service is unhealthy, remove the service_id
                self.service_id = ''
            else:
                raise
        return service

    @gen.coroutine
    def start(self):
        """Start the single-user server in a docker service.
        You can specify the params for the service through jupyterhub_config.py
        or using the user_options
        """
        if self.use_user_options:
            user_options = self.user_options

        else:
            user_options = {}

        self.log.warn("self.use_user_options:".format(self.use_user_options))
        self.log.warn(user_options)

        service = yield self.get_service()

        if service is None:

            if 'name' in user_options:
                self.server_name = user_options['name']

            if hasattr(self, 'container_spec') and self.container_spec is not None:
                container_spec = dict(**self.container_spec)
            elif user_options == {}:
                raise("A container_spec is needed in to create a service")

            container_spec.update(user_options.get('container_spec', {}))

            # iterates over mounts to create
            # a new mounts list of docker.types.Mount
            container_spec['mounts'] = []
            for mount in self.container_spec['mounts']:
                m = dict(**mount)
                if m['type'] == 'volume' and 'source' in m:
                    m['source'] = m['source'].format(username=self.service_owner)
                container_spec['mounts'].append(docker.types.Mount(**m))

            # some Envs are required by the single-user-image
            container_spec['env'] = self.get_env()

            if hasattr(self, 'resource_spec'):
                resource_spec = self.resource_spec
            resource_spec.update(user_options.get('resource_spec', {}))

            if hasattr(self, 'networks'):
                networks = self.networks
            if user_options.get('networks') is not None:
                networks = user_options.get('networks')

            image = container_spec['Image']
            del container_spec['Image']

            # create the service
            container_spec = docker.types.ContainerSpec(image, **container_spec)
            resources = docker.types.Resources(**resource_spec)

            task_spec = {'container_spec': container_spec,
                         'resources': resources,
                         'placement': user_options.get('placement')
                         }
            task_tmpl = docker.types.TaskTemplate(**task_spec)

            resp = yield self.docker('create_service',
                                     task_tmpl,
                                     name=self.service_name,
                                     networks=networks)

            self.service_id = resp['ID']

            self.log.info(
                "Created service '%s' (id: %s) from image %s",
                self.service_name, self.service_id[:7], image)

        else:
            self.log.info(
                "Found existing service '%s' (id: %s)",
                self.service_name, self.service_id[:7])
            # Handle re-using API token.
            # Get the API token from the environment variables
            # of the running service:
            for line in service['Config']['Env']:
                if line.startswith('JPY_API_TOKEN='):
                    self.api_token = line.split('=', 1)[1]
                    break

        ip = self.service_name
        port = self.service_port

        # we use service_name instead of ip
        # https://docs.docker.com/engine/swarm/networking/#use-swarm-mode-service-discovery
        # service_port is actually equal to 8888
        return (ip, port)

    @gen.coroutine
    def stop(self, now=False):
        """Stop and remove the service

        Consider using stop/start when Docker adds support
        """
        self.log.info(
            "Stopping and removing service %s (id: %s)",
            self.service_name, self.service_id[:7])
        yield self.docker('remove_service', self.service_id[:7])
        self.log.info(
            "Service removed %s (id: %s)",
            self.service_name, self.service_id[:7])

        self.clear_state()
<commit_msg>fix server_name<commit_after>"""
A Spawner for JupyterHub that runs each user's
server in a separate Docker Service
"""

import os
import string
import hashlib
from textwrap import dedent
from concurrent.futures import ThreadPoolExecutor
from pprint import pformat

import docker
from docker.errors import APIError
from docker.utils import kwargs_from_env
from tornado import gen

from jupyterhub.spawner import Spawner
from traitlets import (
    Dict,
    Unicode,
    List,
    Bool,
    Int,
)

class UnicodeOrFalse(Unicode):
    info_text = 'a unicode string or False'

    def validate(self, obj, value):
        if value is False:
            return value
        return super(UnicodeOrFalse, self).validate(obj, value)

class SwarmSpawner(Spawner):
    """A Spawner for JupyterHub using Docker Engine in Swarm mode
    """

    _executor = None

    @property
    def executor(self, max_workers=1):
        """single global executor"""
        cls = self.__class__
        if cls._executor is None:
            cls._executor = ThreadPoolExecutor(max_workers)
        return cls._executor

    _client = None

    @property
    def client(self):
        """single global client instance"""
        cls = self.__class__
        if cls._client is None:
            if self.use_docker_client_env:
                kwargs = kwargs_from_env(
                    assert_hostname=self.tls_assert_hostname
                )
                client = docker.APIClient(version='auto', **kwargs)
            else:
                if self.tls:
                    tls_config = True
                elif self.tls_verify or self.tls_ca or self.tls_client:
                    tls_config = docker.tls.TLSConfig(
                        client_cert=self.tls_client,
                        ca_cert=self.tls_ca,
                        verify=self.tls_verify,
                        assert_hostname=self.tls_assert_hostname)
                else:
                    tls_config = None

                docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')

                client = docker.APIClient(base_url=docker_host,
                                          tls=tls_config,
                                          version='auto')
            cls._client = client
        return cls._client

    service_id = Unicode()
    service_port = Int(8888, min=1, max=65535, config=True)
    service_image = Unicode("jupyterhub/singleuser", config=True)
    service_prefix = Unicode(
        "jupyter",
        config=True,
        help=dedent(
            """
            Prefix for service names. The full service name for a particular
            user will be <prefix>-<hash(username)>-<server_name>.
            """
        )
    )

    use_docker_client_env = Bool(False, config=True,
                                 help="If True, will use Docker client env \
                                 variable (boot2docker friendly)")
    tls = Bool(False, config=True, help="If True, connect to docker with --tls")
    tls_verify = Bool(False, config=True, help="If True, connect to docker with --tlsverify")
    tls_ca = Unicode("", config=True, help="Path to CA certificate for docker TLS")
    tls_cert = Unicode("", config=True, help="Path to client certificate for docker TLS")
    tls_key = Unicode("", config=True, help="Path to client key for docker TLS")
    tls_assert_hostname = UnicodeOrFalse(default_value=None,
                                         allow_none=True,
                                         config=True,
                                         help="If False, do not verify hostname of docker daemon",)

    container_spec = Dict({}, config=True, help="Params to create the service")
    resource_spec = Dict({}, config=True, help="Params about cpu and memory limits")
    networks = List([], config=True, help="Additional args to create_host_config for service create")

    _docker_safe_chars = set(string.ascii_letters + string.digits + '-')
    _docker_escape_char = '_'

    use_user_options = Bool(False, config=True, help="the spawner will use the dict passed through the form or as json body when using the Hub Api")
    jupyterhub_service_name = Unicode(config=True,
                                      help=dedent("""Name of the service running the JupyterHub"""))

    @property
    def tls_client(self):
        """A tuple consisting of the TLS client certificate and key if they
        have been provided, otherwise None.

        """
        if self.tls_cert and self.tls_key:
            return (self.tls_cert, self.tls_key)
        return None

    _service_owner = None

    @property
    def service_owner(self):
        if self._service_owner is None:
            m = hashlib.md5()
            m.update(self.user.name.encode('utf-8'))
            self._service_owner = m.hexdigest()
        return self._service_owner

    @property
    def service_name(self):
        """
        Service name inside the Docker Swarm

        service_suffix should be a numerical value unique for user
        {service_prefix}-{service_owner}-{service_suffix}
        """
        # JupyterHub with multi-server-per-user should pass server_name
        # In the future we can remove this part
        server_name = getattr(self, "server_name", None)
        if server_name is None:
            server_name = 1

        return "{}-{}-{}".format(self.service_prefix,
                                 self.service_owner,
                                 server_name
                                )

    def load_state(self, state):
        super().load_state(state)
        self.service_id = state.get('service_id', '')

    def get_state(self):
        state = super().get_state()
        if self.service_id:
            state['service_id'] = self.service_id
        return state

    def _env_keep_default(self):
        """it's called in traitlets. It's a special method name.
        Don't inherit any env from the parent process"""
        return []

    def _public_hub_api_url(self):
        proto, path = self.hub.api_url.split('://', 1)
        _, rest = path.split(':', 1)
        return '{proto}://{name}:{rest}'.format(
            proto=proto,
            name=self.jupyterhub_service_name,
            rest=rest
        )

    def get_env(self):
        env = super().get_env()
        env.update(dict(
            JPY_USER=self.user.name,
            JPY_COOKIE_NAME=self.user.server.cookie_name,
            JPY_BASE_URL=self.user.server.base_url,
            JPY_HUB_PREFIX=self.hub.server.base_url
        ))

        if self.notebook_dir:
            env['NOTEBOOK_DIR'] = self.notebook_dir

        env['JPY_HUB_API_URL'] = self._public_hub_api_url()

        return env

    def _docker(self, method, *args, **kwargs):
        """wrapper for calling docker methods

        to be passed to ThreadPoolExecutor
        """
        m = getattr(self.client, method)
        return m(*args, **kwargs)

    def docker(self, method, *args, **kwargs):
        """Call a docker method in a background thread

        returns a Future
        """
        return self.executor.submit(self._docker, method, *args, **kwargs)

    @gen.coroutine
    def poll(self):
        """Check for a task state like `docker service ps id`"""
        service = yield self.get_service()
        if not service:
            self.log.warn("Service not found")
            return ""

        task_filter = {'service': service['Spec']['Name']}

        task = yield self.docker(
            'tasks', task_filter
        )

        # use the first task and only task
        task = task[0]

        task_state = task['Status']['State']
        self.log.debug(
            "Task %s of Service %s status: %s",
            task['ID'][:7],
            self.service_id[:7],
            pformat(task_state),
        )

        if task_state == 'running':
            return None
        else:
            return (
                "ExitCode={ExitCode}, "
                "Error='{Error}', "
                "FinishedAt={FinishedAt}".format(**task_state)
            )

    @gen.coroutine
    def get_service(self):
        self.log.debug("Getting service '%s'", self.service_name)
        try:
            service = yield self.docker(
                'inspect_service', self.service_name
            )
            self.service_id = service['ID']
        except APIError as err:
            if err.response.status_code == 404:
                self.log.info("Service '%s' is gone", self.service_name)
                service = None
                # Docker service is gone, remove service id
                self.service_id = ''
            elif err.response.status_code == 500:
                self.log.info("Docker Swarm Server error")
                service = None
                # Docker service is unhealthy, remove the service_id
                self.service_id = ''
            else:
                raise
        return service

    @gen.coroutine
    def start(self):
        """Start the single-user server in a docker service.
        You can specify the params for the service through jupyterhub_config.py
        or using the user_options
        """
        if self.use_user_options:
            user_options = self.user_options

        else:
            user_options = {}

        self.log.warn("user_options: ".format(user_options))

        service = yield self.get_service()

        if service is None:

            if 'name' in user_options:
                self.server_name = user_options['name']

            if hasattr(self, 'container_spec') and self.container_spec is not None:
                container_spec = dict(**self.container_spec)
            elif user_options == {}:
                raise("A container_spec is needed in to create a service")

            container_spec.update(user_options.get('container_spec', {}))

            # iterates over mounts to create
            # a new mounts list of docker.types.Mount
            container_spec['mounts'] = []
            for mount in self.container_spec['mounts']:
                m = dict(**mount)
                if m['type'] == 'volume' and 'source' in m:
                    m['source'] = m['source'].format(username=self.service_owner)
                container_spec['mounts'].append(docker.types.Mount(**m))

            # some Envs are required by the single-user-image
            container_spec['env'] = self.get_env()

            if hasattr(self, 'resource_spec'):
                resource_spec = self.resource_spec
            resource_spec.update(user_options.get('resource_spec', {}))

            if hasattr(self, 'networks'):
                networks = self.networks
            if user_options.get('networks') is not None:
                networks = user_options.get('networks')

            image = container_spec['Image']
            del container_spec['Image']

            # create the service
            container_spec = docker.types.ContainerSpec(image, **container_spec)
            resources = docker.types.Resources(**resource_spec)

            task_spec = {'container_spec': container_spec,
                         'resources': resources,
                         'placement': user_options.get('placement')
                         }
            task_tmpl = docker.types.TaskTemplate(**task_spec)

            resp = yield self.docker('create_service',
                                     task_tmpl,
                                     name=self.service_name,
                                     networks=networks)

            self.service_id = resp['ID']

            self.log.info(
                "Created service '%s' (id: %s) from image %s",
                self.service_name, self.service_id[:7], image)

        else:
            self.log.info(
                "Found existing service '%s' (id: %s)",
                self.service_name, self.service_id[:7])
            # Handle re-using API token.
            # Get the API token from the environment variables
            # of the running service:
            for line in service['Config']['Env']:
                if line.startswith('JPY_API_TOKEN='):
                    self.api_token = line.split('=', 1)[1]
                    break

        ip = self.service_name
        port = self.service_port

        # we use service_name instead of ip
        # https://docs.docker.com/engine/swarm/networking/#use-swarm-mode-service-discovery
        # service_port is actually equal to 8888
        return (ip, port)

    @gen.coroutine
    def stop(self, now=False):
        """Stop and remove the service

        Consider using stop/start when Docker adds support
        """
        self.log.info(
            "Stopping and removing service %s (id: %s)",
            self.service_name, self.service_id[:7])
        yield self.docker('remove_service', self.service_id[:7])
        self.log.info(
            "Service removed %s (id: %s)",
            self.service_name, self.service_id[:7])

        self.clear_state()
<|endoftext|>


<commit_before>import time
import os
import hashlib
import copy
import fnmatch

from StringIO import StringIO
from itertools import chain

try:
    import simplejson as json
except ImportError:
    import json

from gevent import monkey
from gevent.pool import Pool
from libcloud.utils.files import exhaust_iterator
from libcloud.storage.base import Container, Object
from libcloud.storage.types import ContainerDoesNotExistError
from libcloud.storage.types import ObjectDoesNotExistError

monkey.patch_all()

from file_syncer.file_lock import FileLock
from file_syncer.constants import MANIFEST_FILE

class FileSyncer(object):
    def __init__(self, directory, provider_cls, username, api_key,
                 container_name, cache_path, exclude_patterns,
                 logger, concurrency=20):
        self._directory = directory
        self._provider_cls = provider_cls
        self._username = username
        self._api_key = api_key
        self._container_name = container_name
        self._cache_path = cache_path
        self._exclude_patterns = exclude_patterns
        self._logger = logger
        self._concurrency = concurrency

        self._uploaded = []
        self._removed = []

        if not os.path.exists(self._directory):
            raise ValueError('Directory %s doesn\'t exist' %
                             (self._directory))

        self._logger.info('Using provider: %(name)s',
                          {'name': provider_cls.name})

        self._setup_cache_path()
        self._setup_container()

    def _setup_cache_path(self):
        """
        Create a local cache directory, if it doesn't already exist.
        """
        if not os.path.exists(self._cache_path):
            self._logger.debug('Cache directory doesn\'t exist ' +
              '(%(directory)s), creating it...', {'directory': self._cache_path})
            os.makedirs(self._cache_path)

    def _setup_container(self):
        """
        Create a container if it doesn't already exist.
        """
        driver = self._get_driver_instance()

        try:
            container = driver.get_container(container_name=self._container_name)
        except ContainerDoesNotExistError:
            self._logger.debug('Container "%(name)s" doesn\'t exist, ' +
                    'creating it..', {'name': self._container_name})
            container = driver.create_container(container_name=self._container_name)

        self._container = container

    def _get_driver_instance(self):
        driver = self._provider_cls(self._username, self._api_key)
        return driver

    def _include_file(self, file_name):
        """
        Return True if the file should be included, False otherwise.
        """
        for pattern in self._exclude_patterns:
            if fnmatch.fnmatch(file_name, pattern):
                return

        return True

    def sync(self):
        """
        Synchronizes remote directory with a local one.
        """
        digest = hashlib.md5(self._directory).hexdigest()
        lock_file_path = os.path.join(digest)

        pool = Pool(self._concurrency)

        with FileLock(lock_file_path, timeout=None):
            # Ensure that only a single process runs at the same time
            time_start = time.time()
            local_files = self._get_local_files(directory=self._directory)
            self._logger.debug('Found %(count)s local files',
                               {'count': len(local_files)})

            remote_files = self._get_remote_files()
            self._logger.debug('Found %(count)s remote files',
                               {'count': len(remote_files)})

            differences = self._get_differences(local_files=local_files,
                                                remote_files=remote_files)
            actions = self._calculate_actions(differences=differences)

            self._logger.info('To remove: %(to_remove)s, to upload: %(to_upload)s',
                              {'to_remove': len(actions['to_remove']),
                               'to_upload': len(actions['to_upload'])})

            # Synchronization is performed in two steps:
            # 1 - Upload new or changed files and remove deleted ones
            # 2 - Upload manifest

            for item in actions['to_remove']:
                func = lambda item: self._remove_object(item=item)
                pool.spawn(func, item)

            for item in actions['to_upload']:
                func = lambda item: self._upload_object(item=item)
                pool.spawn(func, item)

            pool.join()

            manifest = self._generate_manifest(remote_files=remote_files)
            self._upload_manifest(json.dumps(manifest))

            took = (time.time() - time_start)
            self._logger.info('Synchronization complete, took: %(took)0.2f' +
                              ' seconds', {'took': took})

    def restore(self):
        """
        Restores a remote container to the file system
        """
        digest = hashlib.md5(self._directory).hexdigest()
        lock_file_path = os.path.join(digest)

        pool = Pool(self._concurrency)
        with FileLock(lock_file_path, timeout=None):
            # Ensure that only a single process runs at the same time
            time_start = time.time()

            for item in self._get_remote_files():
                func = lambda item: self._download_remote_file(name=item)
                pool.spawn(func, item)

            pool.join()

            took = (time.time() - time_start)
            self._logger.info('Synchronization complete, took: %(took)0.2f' +
                              ' seconds', {'took': took})

    def _get_item_remote_name(self, name, file_path):
        return file_path.replace(self._directory, '')

    def _generate_manifest(self, remote_files):
        manifest = copy.deepcopy(remote_files)

        for item in self._uploaded:
            manifest[item['remote_name']] = item

        for item in self._removed:
            if item['remote_name'] in manifest:
                del manifest[item['remote_name']]

        return manifest

    def _upload_manifest(self, data):
        driver = self._get_driver_instance()
        name = MANIFEST_FILE
        extra = {'content_type': 'application/json'}
        container = Container(name=self._container_name, extra=extra, driver=driver)
        iterator = StringIO(data)
        driver.upload_object_via_stream(iterator=iterator, container=container,
                                        object_name=name)

    def _remove_object(self, item):
        driver = self._get_driver_instance()
        name = item['remote_name']

        self._logger.debug('Removing object: %(name)s', {'name': name})

        container = Container(name=self._container_name, extra={}, driver=driver)
        obj = Object(name=name, size=None, hash=None, extra=None,
                     meta_data=None, container=container, driver=driver)

        try:
            driver.delete_object(obj=obj)
        except Exception, e:
            self._logger.error('Failed to remove object "%(name)s": %(error)s',
                               {'name': name, 'error': str(e)})
            return

        self._removed.append(item)
        self._logger.debug('Object removed: %(name)s', {'name': name})

    def _upload_object(self, item):
        driver = self._get_driver_instance()
        name = item['remote_name']
        file_path = item['path']

        self._logger.debug('Uploading object: %(name)s', {'name': name})

        extra = {'content_type': 'application/octet-stream'}
        container = Container(name=self._container_name, extra=None, driver=driver)

        try:
            driver.upload_object(file_path=file_path, container=container,
                                 object_name=name, extra=extra)
        except Exception, e:
            self._logger.error('Failed to upload object "%(name)s": %(error)s',
                               {'name': name, 'error': str(e)})
            return

        self._uploaded.append(item)
        self._logger.debug('Object uploaded: %(name)s', {'name': name})

    def _get_local_files(self, directory):
        """
        Recursively find all the files in a directory.

        @rtype C{dict}
        """
        result = {}

        base_path = os.path.abspath(directory)
        for (dirpath, dirnames, filenames) in os.walk(directory):
            for name in filenames:

                file_path = os.path.join(base_path, dirpath, name)
                remote_name = self._get_item_remote_name(name=name,
                                                         file_path=file_path)

                if not self._include_file(remote_name):
                    self._logger.debug('File %(name)s is excluded, skipping it',
                                       {'name': name})
                    continue

                mtime = os.path.getmtime(file_path)
                md5_hash = None

                item = {'name': name, 'remote_name': remote_name, 'path': file_path,
                        'last_modified': mtime, 'md5_hash': md5_hash}
                result[remote_name] = item

        return result

    def _get_remote_files(self):
        """
        Return a list of files in a container.
        """
        driver = self._get_driver_instance()

        try:
            obj = driver.get_object(container_name=self._container_name,
                                    object_name=MANIFEST_FILE)
        except ObjectDoesNotExistError:
            self._logger.debug('Manifest doesn\'t exist, assuming that ' +
                               'there are no remote files')
            return {}

        iterator = driver.download_object_as_stream(obj=obj)
        data = exhaust_iterator(iterator=iterator)

        try:
            parsed = json.loads(data)
        except Exception, e:
            raise Exception('Corrupted manifest, failed to parse it: ' + str(e))

        return parsed

    def _download_remote_file(self, name):
        """
        Return a list of files in a container.
        """

        self._logger.debug('Downloading object: %(name)s to %(path)s',
                {'name': name, 'path': self._directory})

        # strip the leading slash if it exists in the object_name
        local_filename = name
        if local_filename[0] == '/':
            local_filename = local_filename[1:]

        driver = self._get_driver_instance()
        filepath = os.path.join(self._directory, local_filename)

        try:
            obj = driver.get_object(container_name=self._container_name,
                                    object_name=name)
        except ObjectDoesNotExistError:
            self._logger.debug('Object ' + name + ' doesn\'t exist')
            return

        driver.download_object(obj=obj, destination_path=filepath,
                overwrite_existing=True, delete_on_failure=True)

    def _get_differences(self, local_files, remote_files):
        """
        Return differences between a local and remote copy.

        @rtype C{dict} A dictionary with the following keys:

        added - files which have been added locally.
        removed - files which have been removed.
        modified - files which have been modified.
        """
        result = {'added': {}, 'removed': {}, 'modified': {}}

        for name, local_item in local_files.iteritems():
            remote_item = remote_files.get(name, None)

            if remote_item is None:
                # New file
                result['added'][name] = local_item
            elif local_item['last_modified'] > remote_item['last_modified']:
                # Local file has been modified
                result['modified'][name] = local_item

        for name, remote_item in remote_files.iteritems():
            name = remote_item['remote_name']
            local_item = local_files.get(name, None)

            if not local_item:
                # File has been deleted locally
                result['removed'][name] = remote_item

        return result

    def _calculate_actions(self, differences):
        """
        Return actions which need to be performed to make the remote copy match
        a local one.
        """
        result = {'to_upload': [], 'to_remove': []}

        for item in chain(differences['added'].values(),
                          differences['modified'].values()):
            result['to_upload'].append(item)

        for item in differences['removed'].values():
            result['to_remove'].append(item)

        return result
<commit_msg>fix docstring<commit_after>import time
import os
import hashlib
import copy
import fnmatch

from StringIO import StringIO
from itertools import chain

try:
    import simplejson as json
except ImportError:
    import json

from gevent import monkey
from gevent.pool import Pool
from libcloud.utils.files import exhaust_iterator
from libcloud.storage.base import Container, Object
from libcloud.storage.types import ContainerDoesNotExistError
from libcloud.storage.types import ObjectDoesNotExistError

monkey.patch_all()

from file_syncer.file_lock import FileLock
from file_syncer.constants import MANIFEST_FILE

class FileSyncer(object):
    def __init__(self, directory, provider_cls, username, api_key,
                 container_name, cache_path, exclude_patterns,
                 logger, concurrency=20):
        self._directory = directory
        self._provider_cls = provider_cls
        self._username = username
        self._api_key = api_key
        self._container_name = container_name
        self._cache_path = cache_path
        self._exclude_patterns = exclude_patterns
        self._logger = logger
        self._concurrency = concurrency

        self._uploaded = []
        self._removed = []

        if not os.path.exists(self._directory):
            raise ValueError('Directory %s doesn\'t exist' %
                             (self._directory))

        self._logger.info('Using provider: %(name)s',
                          {'name': provider_cls.name})

        self._setup_cache_path()
        self._setup_container()

    def _setup_cache_path(self):
        """
        Create a local cache directory, if it doesn't already exist.
        """
        if not os.path.exists(self._cache_path):
            self._logger.debug('Cache directory doesn\'t exist ' +
              '(%(directory)s), creating it...', {'directory': self._cache_path})
            os.makedirs(self._cache_path)

    def _setup_container(self):
        """
        Create a container if it doesn't already exist.
        """
        driver = self._get_driver_instance()

        try:
            container = driver.get_container(container_name=self._container_name)
        except ContainerDoesNotExistError:
            self._logger.debug('Container "%(name)s" doesn\'t exist, ' +
                    'creating it..', {'name': self._container_name})
            container = driver.create_container(container_name=self._container_name)

        self._container = container

    def _get_driver_instance(self):
        driver = self._provider_cls(self._username, self._api_key)
        return driver

    def _include_file(self, file_name):
        """
        Return True if the file should be included, False otherwise.
        """
        for pattern in self._exclude_patterns:
            if fnmatch.fnmatch(file_name, pattern):
                return

        return True

    def sync(self):
        """
        Synchronizes remote directory with a local one.
        """
        digest = hashlib.md5(self._directory).hexdigest()
        lock_file_path = os.path.join(digest)

        pool = Pool(self._concurrency)

        with FileLock(lock_file_path, timeout=None):
            # Ensure that only a single process runs at the same time
            time_start = time.time()
            local_files = self._get_local_files(directory=self._directory)
            self._logger.debug('Found %(count)s local files',
                               {'count': len(local_files)})

            remote_files = self._get_remote_files()
            self._logger.debug('Found %(count)s remote files',
                               {'count': len(remote_files)})

            differences = self._get_differences(local_files=local_files,
                                                remote_files=remote_files)
            actions = self._calculate_actions(differences=differences)

            self._logger.info('To remove: %(to_remove)s, to upload: %(to_upload)s',
                              {'to_remove': len(actions['to_remove']),
                               'to_upload': len(actions['to_upload'])})

            # Synchronization is performed in two steps:
            # 1 - Upload new or changed files and remove deleted ones
            # 2 - Upload manifest

            for item in actions['to_remove']:
                func = lambda item: self._remove_object(item=item)
                pool.spawn(func, item)

            for item in actions['to_upload']:
                func = lambda item: self._upload_object(item=item)
                pool.spawn(func, item)

            pool.join()

            manifest = self._generate_manifest(remote_files=remote_files)
            self._upload_manifest(json.dumps(manifest))

            took = (time.time() - time_start)
            self._logger.info('Synchronization complete, took: %(took)0.2f' +
                              ' seconds', {'took': took})

    def restore(self):
        """
        Restores a remote container to the file system
        """
        digest = hashlib.md5(self._directory).hexdigest()
        lock_file_path = os.path.join(digest)

        pool = Pool(self._concurrency)
        with FileLock(lock_file_path, timeout=None):
            # Ensure that only a single process runs at the same time
            time_start = time.time()

            for item in self._get_remote_files():
                func = lambda item: self._download_remote_file(name=item)
                pool.spawn(func, item)

            pool.join()

            took = (time.time() - time_start)
            self._logger.info('Synchronization complete, took: %(took)0.2f' +
                              ' seconds', {'took': took})

    def _get_item_remote_name(self, name, file_path):
        return file_path.replace(self._directory, '')

    def _generate_manifest(self, remote_files):
        manifest = copy.deepcopy(remote_files)

        for item in self._uploaded:
            manifest[item['remote_name']] = item

        for item in self._removed:
            if item['remote_name'] in manifest:
                del manifest[item['remote_name']]

        return manifest

    def _upload_manifest(self, data):
        driver = self._get_driver_instance()
        name = MANIFEST_FILE
        extra = {'content_type': 'application/json'}
        container = Container(name=self._container_name, extra=extra, driver=driver)
        iterator = StringIO(data)
        driver.upload_object_via_stream(iterator=iterator, container=container,
                                        object_name=name)

    def _remove_object(self, item):
        driver = self._get_driver_instance()
        name = item['remote_name']

        self._logger.debug('Removing object: %(name)s', {'name': name})

        container = Container(name=self._container_name, extra={}, driver=driver)
        obj = Object(name=name, size=None, hash=None, extra=None,
                     meta_data=None, container=container, driver=driver)

        try:
            driver.delete_object(obj=obj)
        except Exception, e:
            self._logger.error('Failed to remove object "%(name)s": %(error)s',
                               {'name': name, 'error': str(e)})
            return

        self._removed.append(item)
        self._logger.debug('Object removed: %(name)s', {'name': name})

    def _upload_object(self, item):
        driver = self._get_driver_instance()
        name = item['remote_name']
        file_path = item['path']

        self._logger.debug('Uploading object: %(name)s', {'name': name})

        extra = {'content_type': 'application/octet-stream'}
        container = Container(name=self._container_name, extra=None, driver=driver)

        try:
            driver.upload_object(file_path=file_path, container=container,
                                 object_name=name, extra=extra)
        except Exception, e:
            self._logger.error('Failed to upload object "%(name)s": %(error)s',
                               {'name': name, 'error': str(e)})
            return

        self._uploaded.append(item)
        self._logger.debug('Object uploaded: %(name)s', {'name': name})

    def _get_local_files(self, directory):
        """
        Recursively find all the files in a directory.

        @rtype C{dict}
        """
        result = {}

        base_path = os.path.abspath(directory)
        for (dirpath, dirnames, filenames) in os.walk(directory):
            for name in filenames:

                file_path = os.path.join(base_path, dirpath, name)
                remote_name = self._get_item_remote_name(name=name,
                                                         file_path=file_path)

                if not self._include_file(remote_name):
                    self._logger.debug('File %(name)s is excluded, skipping it',
                                       {'name': name})
                    continue

                mtime = os.path.getmtime(file_path)
                md5_hash = None

                item = {'name': name, 'remote_name': remote_name, 'path': file_path,
                        'last_modified': mtime, 'md5_hash': md5_hash}
                result[remote_name] = item

        return result

    def _get_remote_files(self):
        """
        Return a list of files in a container.
        """
        driver = self._get_driver_instance()

        try:
            obj = driver.get_object(container_name=self._container_name,
                                    object_name=MANIFEST_FILE)
        except ObjectDoesNotExistError:
            self._logger.debug('Manifest doesn\'t exist, assuming that ' +
                               'there are no remote files')
            return {}

        iterator = driver.download_object_as_stream(obj=obj)
        data = exhaust_iterator(iterator=iterator)

        try:
            parsed = json.loads(data)
        except Exception, e:
            raise Exception('Corrupted manifest, failed to parse it: ' + str(e))

        return parsed

    def _download_remote_file(self, name):
        """
        Download a remote file given a name.
        """

        self._logger.debug('Downloading object: %(name)s to %(path)s',
                {'name': name, 'path': self._directory})

        # strip the leading slash if it exists in the object_name
        local_filename = name
        if local_filename[0] == '/':
            local_filename = local_filename[1:]

        driver = self._get_driver_instance()
        filepath = os.path.join(self._directory, local_filename)

        try:
            obj = driver.get_object(container_name=self._container_name,
                                    object_name=name)
        except ObjectDoesNotExistError:
            self._logger.debug('Object ' + name + ' doesn\'t exist')
            return

        driver.download_object(obj=obj, destination_path=filepath,
                overwrite_existing=True, delete_on_failure=True)

    def _get_differences(self, local_files, remote_files):
        """
        Return differences between a local and remote copy.

        @rtype C{dict} A dictionary with the following keys:

        added - files which have been added locally.
        removed - files which have been removed.
        modified - files which have been modified.
        """
        result = {'added': {}, 'removed': {}, 'modified': {}}

        for name, local_item in local_files.iteritems():
            remote_item = remote_files.get(name, None)

            if remote_item is None:
                # New file
                result['added'][name] = local_item
            elif local_item['last_modified'] > remote_item['last_modified']:
                # Local file has been modified
                result['modified'][name] = local_item

        for name, remote_item in remote_files.iteritems():
            name = remote_item['remote_name']
            local_item = local_files.get(name, None)

            if not local_item:
                # File has been deleted locally
                result['removed'][name] = remote_item

        return result

    def _calculate_actions(self, differences):
        """
        Return actions which need to be performed to make the remote copy match
        a local one.
        """
        result = {'to_upload': [], 'to_remove': []}

        for item in chain(differences['added'].values(),
                          differences['modified'].values()):
            result['to_upload'].append(item)

        for item in differences['removed'].values():
            result['to_remove'].append(item)

        return result
<|endoftext|>


<commit_before>import pandas as pd
import statsmodels.api as sm
import pylab as pl
import numpy as np

# Our own plotting functions, somewhat specific to this hands-on session.
from plots import *

# Read the data (edit the path if necessary)
# The data file has no row indices/names, hence index_col=F.
df = pd.read_csv("wine_quality_data.csv", index_col=False)

# How the data looks like... first rows.
print df.head()

# What columns/variables are there?
print df.columns

# Variables that could be used to predict the wine type.
def chemvars(df):
	return df[['fixed_acidity', 'volatile_acidity', 'citric_acid',
		  'residual_sugar', 'chlorides', 'free_sulfur_dioxide',
		  'total_sulfur_dioxide', 'density', 'pH', 'sulphates',
		  'alcohol']]
# Yet another way to look at the data.
print chemvars(df).describe()

# Histograms of all predictors.
plot_histograms(chemvars(df))

# The wine type is a string. Make a boolean type indicator.
def is_red(df): return np.array(df['wine_color']=="red", int)

# Calling the thing to be predicted 'y' is a convention.
y = is_red(df)

# Histograms by the wine type.
# Hint: This is very useful when you select or transform variables for a model
# later.
plot_class_hists(y, chemvars(df), alpha=.3)

# Pandas data frames are handy for calculating class-wise summary statistics.
# Some examples...
# - Percentages of wine types in the data
print 100*df.groupby(y).size()/float(len(df))
# - Means (averages) by wine type
print df.groupby(y).mean()
# - Standard deviations by wine type
print df.groupby(y).std()

# We could use all the predictors in a model.
# But to keep things simple, let's start with two variables only.
# (Things are easier to plot if you have only two dimensions.)
# Choose variables that separate the target classes (wine type) well.
# For example...
# Hint: Try other predictors. Look at the histograms by wine type.

# Plot scatter plot of the chosen vars, colors specify wine type.
plot_scatter(y, chemvars(df), 'fixed_acidity', 'chlorides', alpha=0.2, ymax=0.4)
# Our function allows transformations, see the code in plots.py
# Logarithm compresses high values and opens up the scale at the lower end.
plot_scatter(y, chemvars(df), 'fixed_acidity', 'chlorides', np.log, np.log, alpha=0.2, ymax=0.4)

# Modeling starts here...

# Fit a logistic regression model using two variables.
# [FIXME: add a reference somewhere, a slide or Wikipedia or something]
# In a regression model, you should have a free constant in the linear combination.
# It is called intercept.
# (Think for a while a logistic regression with no variables, but either with an intercept
#  or without. The latter always gives p=0.5, the former adjusts to the prior class probabilities
#  of the data.)
# The intercept becomes conveniently added if you add a constant variable to the data.
# The function returns predictors for chosen two variables:
def preds2(df, v1, v2):
	X = df[[v1, v2]]
	X['intercept'] = 1.0
	return X

# For convenience, define a list of variables we will use.
# (Note: tuple does not work here because this is used for indexing later)

# The magic happens here.
X = preds2(df, 'fixed_acidity', 'chlorides')
model = sm.Logit(y, X)			# Create model object.
mfit = model.fit()			# Train the model, i.e., optimize its parameters
print mfit.summary()			# It did something!

# Predictions: what the model "remembers" about the wines in the training data,
# are the wines red or white (probabilities for the wine being red)
p = mfit.predict(X)
# How those probabilities compare to what the wines actually were?
# There are many ways to evaluate the predictions, here we just
# see how many hits are correct if probabilities over 0.5 are guessed as red.
def pacc(y, p):
	return 100*sum(y == (p>0.5)) / float(len(y))
print pacc(y, p)

# From the model's point of view, wine is what we have shown about it: vars2d.
# Because of the '2d', it is easy to plot what the model thinks about
# all possible wines, that is, all possible values of vars2d.
# (remember, 'intercept' is just a constant)
# Note the zoom tool in the fig!
visualize_result_2d_linear(y, X, mfit, 'fixed_acidity', 'chlorides')

# Shades of gray are probabilities.
# (You may want to look at the plotting code in plots.py, but it is not relevant
# to the big picture.)

# Note the linear decision boundary above. This comes from the linearity of the
# logistic regression model.

# If you want a more flexible decision boundary, you can add nonlinear terms
# to the model.
# Then the model is still linear with respect to the variables it sees, but not
# with respect to the original data!
def add_mterm(X, vars):
    """Add a multiplicative term to the model."""
    X = X.copy()
    X['*'.join(vars)] = np.multiply.reduce(X[list(vars)].values, 1)
    return X

# Here are some multiplicative terms.
# For example, (var1, var1) will add the term var1*var1 to the model,
# so the model would be var1 + var2 + var1*var2 + intercept
def nonlins(v1, v2):
	# A bunch of simple nonlinear multiplicative terms.
	return [(v1, v1), (v2, v2), (v1, v1, v1), (v2, v2, v2), (v1, v2)]
# Let's make a data with all the nonlinear terms above!
v1, v2 = 'fixed_acidity', 'chlorides'
Xn = reduce(add_mterm, nonlins(v1, v2), preds2(df, v1, v2))
y = is_red(df)				# (just as a reminder)

# Fit the model with all the new nonlinear terms.
m_nl = sm.Logit(y, Xn) 			# Model object with the expanded set of predictors.
mfit_nl = m_nl.fit()			# Fit it.
print mfit_nl.summary()			# See how it is

# Probabilities the new model gives to wines being red
# Hint: you may plot p_nl against the predictions of the linear model (p).
p_nl = mfit_nl.predict(Xn)
# How many does it get right, in percentages?
print pacc(y, p_nl)

# Visualize the probabilities (decision boundary) of the nonlinear model.
# This relies on v1, v2, and mfit defined above.
# - Set up a grid of values of the original variables
x1g, x2g = np.meshgrid(vargrid(df[v1]), vargrid(df[v2]))
# - Make a data frame out of the values on the grid.
Xg = pd.DataFrame({v1 : np.ravel(x1g), v2: np.ravel(x2g), 'intercept': 1.0})
# - Expand the data frame by computing all the nonlinearities.
Xg = reduce(add_mterm, nonlins(v1, v2), Xg)
# - Make sure the order of the terms is as in the original data
Xg = Xg[Xn.columns]
# - Compute predictions
pg = mfit_nl.predict(Xg)

# - Plot them
h = pl.contourf(x1g, x2g, np.reshape(pg, x1g.shape), 300, cmap=pl.cm.gist_yarg)
cbar = pl.colorbar()
cbar.set_label('probability for y=1')
# - Overlay the data on top of predictions
pl.plot(X[v1][y==0], X[v2][y==0], 'g^', markeredgecolor='g', alpha=0.5)
pl.plot(X[v1][y==1], X[v2][y==1], 'r*', markeredgecolor='r', alpha=0.5)
pl.xlabel(v1); pl.ylabel(v2)
pl.show()
# Quite a complex decision surface!
# Again, note the zoom tool in the plot window.

# (Last session)
# Evaluating models with leave-out data

# Make this a bit more functional. :)

# Split the data set randomly into two.
df_train, df_test = [i[1] for i in df.groupby(np.random.random(len(df))>.5)]

def my_preds2(df): return preds2(df, 'fixed_acidity', 'chlorides')

model = sm.Logit(is_red(df_train), my_preds2(df_train))
mfit = model.fit()
print mfit.summary()

# Predict on training data
p_train = mfit.predict(my_preds2(df_train))
print pacc(is_red(df_train), p_train)

# Predict on *test* data. The model has never seen these samples.
p_test = mfit.predict(my_preds2(df_test))
print pacc(is_red(df_test), p_test)

# ... or both
print [pacc(is_red(df_part), mfit.predict(my_preds2(df_part))) for df_part in (df_train, df_test)]

# Hmm, how about with all variables?
def preds_all(df):
	X = chemvars(df)
	X['intercept'] = 1.0
	return X

model = sm.Logit(is_red(df_train), preds_all(df_train))
mfit = model.fit()
print [pacc(is_red(df_part), mfit.predict(preds_all(df_part))) for df_part in (df_train, df_test)]

# Make it a function
def test_model(predfun, df_train=df_train, df_test=df_test):
	mfit = sm.Logit(is_red(df_train), predfun(df_train)).fit()
	print [pacc(is_red(df_part), mfit.predict(predfun(df_part))) for df_part in (df_train, df_test)]

print test_model(my_preds2)
print test_model(preds_all)
v1, v2 = 'fixed_acidity', 'chlorides'
print test_model(lambda df: reduce(add_mterm, nonlins(v1, v2), preds2(df, v1, v2)))
print test_model(lambda df: reduce(add_mterm, nonlins(v1, v2), preds_all(df)))

def all_pairs(x): return [(i, j) for i in x for j in x if i<=j]

Xbig = reduce(add_mterm, all_pairs(chemvars(df).columns), chemvars(df))
print Xbig.shape
# Oops, a singular matrix!
print test_model(lambda df: reduce(add_mterm, all_pairs(chemvars(df).columns), chemvars(df)))

# Adding noise helps?
df2 = df.copy()
for v in chemvars(df).columns: df2[v] += np.std(df2[v])*np.random.standard_normal(len(df2))
df_train, df_test = [i[1] for i in df2.groupby(np.random.random(len(df2))>.5)]
def test_model(predfun, df_train=df_train, df_test=df_test):
	mfit = sm.Logit(is_red(df_train), predfun(df_train)).fit()
	print [pacc(is_red(df_part), mfit.predict(predfun(df_part))) for df_part in (df_train, df_test)]
print test_model(my_preds2)
print test_model(preds_all)
v1, v2 = 'fixed_acidity', 'chlorides'
print test_model(lambda df: reduce(add_mterm, nonlins(v1, v2), preds2(df, v1, v2)))
print test_model(lambda df: reduce(add_mterm, nonlins(v1, v2), preds_all(df)))

def all_pairs(x): return [(i, j) for i in x for j in x if i<=j]

Xbig = reduce(add_mterm, all_pairs(chemvars(df).columns), chemvars(df))
print Xbig.shape
# Oops, a singular matrix!
print test_model(lambda df: reduce(add_mterm, all_pairs(chemvars(df).columns), chemvars(df)))

<commit_msg>Removed stuff at the end, added cv.<commit_after>import pandas as pd
import statsmodels.api as sm
import pylab as pl
import numpy as np

# Our own plotting functions, somewhat specific to this hands-on session.
from plots import *

# Read the data (edit the path if necessary)
# The data file has no row indices/names, hence index_col=F.
df = pd.read_csv("wine_quality_data.csv", index_col=False)

# How the data looks like... first rows.
print df.head()

# What columns/variables are there?
print df.columns

# Variables that could be used to predict the wine type.
def chemvars(df):
	return df[['fixed_acidity', 'volatile_acidity', 'citric_acid',
		  'residual_sugar', 'chlorides', 'free_sulfur_dioxide',
		  'total_sulfur_dioxide', 'density', 'pH', 'sulphates',
		  'alcohol']]
# Yet another way to look at the data.
print chemvars(df).describe()

# Histograms of all predictors.
plot_histograms(chemvars(df))

# The wine type is a string. Make a boolean type indicator.
def is_red(df): return np.array(df['wine_color']=="red", int)

# Calling the thing to be predicted 'y' is a convention.
y = is_red(df)

# Histograms by the wine type.
# Hint: This is very useful when you select or transform variables for a model
# later.
plot_class_hists(y, chemvars(df), alpha=.3)

# Pandas data frames are handy for calculating class-wise summary statistics.
# Some examples...
# - Percentages of wine types in the data
print 100*df.groupby(y).size()/float(len(df))
# - Means (averages) by wine type
print df.groupby(y).mean()
# - Standard deviations by wine type
print df.groupby(y).std()

# We could use all the predictors in a model.
# But to keep things simple, let's start with two variables only.
# (Things are easier to plot if you have only two dimensions.)
# Choose variables that separate the target classes (wine type) well.
# For example...
# Hint: Try other predictors. Look at the histograms by wine type.

# Plot scatter plot of the chosen vars, colors specify wine type.
plot_scatter(y, chemvars(df), 'fixed_acidity', 'chlorides', alpha=0.2, ymax=0.4)
# Our function allows transformations, see the code in plots.py
# Logarithm compresses high values and opens up the scale at the lower end.
plot_scatter(y, chemvars(df), 'fixed_acidity', 'chlorides', np.log, np.log, alpha=0.2, ymax=0.4)

# Modeling starts here...

# Fit a logistic regression model using two variables.
# [FIXME: add a reference somewhere, a slide or Wikipedia or something]
# In a regression model, you should have a free constant in the linear combination.
# It is called intercept.
# (Think for a while a logistic regression with no variables, but either with an intercept
#  or without. The latter always gives p=0.5, the former adjusts to the prior class probabilities
#  of the data.)
# The intercept becomes conveniently added if you add a constant variable to the data.
# The function returns predictors for chosen two variables:
def preds2(df, v1, v2):
	X = df[[v1, v2]]
	X['intercept'] = 1.0
	return X

# For convenience, define a list of variables we will use.
# (Note: tuple does not work here because this is used for indexing later)

# The magic happens here.
X = preds2(df, 'fixed_acidity', 'chlorides')
model = sm.Logit(y, X)			# Create model object.
mfit = model.fit()			# Train the model, i.e., optimize its parameters
print mfit.summary()			# It did something!

# Predictions: what the model "remembers" about the wines in the training data,
# are the wines red or white (probabilities for the wine being red)
p = mfit.predict(X)
# How those probabilities compare to what the wines actually were?
# There are many ways to evaluate the predictions, here we just
# see how many hits are correct if probabilities over 0.5 are guessed as red.
def pacc(y, p):
	return 100*sum(y == (p>0.5)) / float(len(y))
print pacc(y, p)

# From the model's point of view, wine is what we have shown about it: vars2d.
# Because of the '2d', it is easy to plot what the model thinks about
# all possible wines, that is, all possible values of vars2d.
# (remember, 'intercept' is just a constant)
# Note the zoom tool in the fig!
visualize_result_2d_linear(y, X, mfit, 'fixed_acidity', 'chlorides')

# Shades of gray are probabilities.
# (You may want to look at the plotting code in plots.py, but it is not relevant
# to the big picture.)

# Note the linear decision boundary above. This comes from the linearity of the
# logistic regression model.

# If you want a more flexible decision boundary, you can add nonlinear terms
# to the model.
# Then the model is still linear with respect to the variables it sees, but not
# with respect to the original data!
def add_mterm(X, vars):
    """Add a multiplicative term to the model."""
    X = X.copy()
    X['*'.join(vars)] = np.multiply.reduce(X[list(vars)].values, 1)
    return X

# Here are some multiplicative terms.
# For example, (var1, var1) will add the term var1*var1 to the model,
# so the model would be var1 + var2 + var1*var2 + intercept
def nonlins(v1, v2):
	# A bunch of simple nonlinear multiplicative terms.
	return [(v1, v1), (v2, v2), (v1, v1, v1), (v2, v2, v2), (v1, v2)]
# Let's make a data with all the nonlinear terms above!
v1, v2 = 'fixed_acidity', 'chlorides'
Xn = reduce(add_mterm, nonlins(v1, v2), preds2(df, v1, v2))
y = is_red(df)				# (just as a reminder)

# Fit the model with all the new nonlinear terms.
m_nl = sm.Logit(y, Xn) 			# Model object with the expanded set of predictors.
mfit_nl = m_nl.fit()			# Fit it.
print mfit_nl.summary()			# See how it is

# Probabilities the new model gives to wines being red
# Hint: you may plot p_nl against the predictions of the linear model (p).
p_nl = mfit_nl.predict(Xn)
# How many does it get right, in percentages?
print pacc(y, p_nl)

# Visualize the probabilities (decision boundary) of the nonlinear model.
# This relies on v1, v2, and mfit defined above.
# - Set up a grid of values of the original variables
x1g, x2g = np.meshgrid(vargrid(df[v1]), vargrid(df[v2]))
# - Make a data frame out of the values on the grid.
Xg = pd.DataFrame({v1 : np.ravel(x1g), v2: np.ravel(x2g), 'intercept': 1.0})
# - Expand the data frame by computing all the nonlinearities.
Xg = reduce(add_mterm, nonlins(v1, v2), Xg)
# - Make sure the order of the terms is as in the original data
Xg = Xg[Xn.columns]
# - Compute predictions
pg = mfit_nl.predict(Xg)

# - Plot them
h = pl.contourf(x1g, x2g, np.reshape(pg, x1g.shape), 300, cmap=pl.cm.gist_yarg)
cbar = pl.colorbar()
cbar.set_label('probability for y=1')
# - Overlay the data on top of predictions
pl.plot(X[v1][y==0], X[v2][y==0], 'g^', markeredgecolor='g', alpha=0.5)
pl.plot(X[v1][y==1], X[v2][y==1], 'r*', markeredgecolor='r', alpha=0.5)
pl.xlabel(v1); pl.ylabel(v2)
pl.show()
# Quite a complex decision surface!
# Again, note the zoom tool in the plot window.

# (Last session)
# Evaluating models with leave-out data

# Make this a bit more functional. :)

# Split the data set randomly into two.
df_train, df_test = [i[1] for i in df.groupby(np.random.random(len(df))>.5)]

def my_preds2(df): return preds2(df, 'fixed_acidity', 'chlorides')

model = sm.Logit(is_red(df_train), my_preds2(df_train))
mfit = model.fit()
print mfit.summary()

# Predict on training data
p_train = mfit.predict(my_preds2(df_train))
print pacc(is_red(df_train), p_train)

# Predict on *test* data. The model has never seen these samples.
p_test = mfit.predict(my_preds2(df_test))
print pacc(is_red(df_test), p_test)

# ... or both
print [pacc(is_red(d), mfit.predict(my_preds2(d))) for d in (df_train, df_test)]

# Hmm, how about with all variables?
def preds_all(df):
	X = chemvars(df)
	X['intercept'] = 1.0
	return X

# You may experiment with more quadratic terms
def all_pairs(x): return [(i, j) for i in x for j in x if i<=j]
df_huge = reduce(add_mterm, all_pairs(chemvars(df).columns), chemvars(df))

# Cross validation does not waste half of the data for testing:
# It splits the data set into (e.g.) 10 pieces, trains on 9, tests on the remaining one,
# and loops this through all the ten pieces.

from crossvalidation import cv

n_success = cv(is_red(df), my_preds2(df), 10,
	       # The function to build the model and predict
	       lambda ytr, Xtr, Xev: sm.Logit(ytr, Xtr).fit().predict(Xev),
	       # The function to evaluate results: How many are right?
	       lambda p, y: sum(y == (p>0.5)))
print 100*n_success/float(len(df))

<|endoftext|>


<commit_before>import unittest
from topo.tk import *
from topo.base.simulator import Simulator
from topo.plotting.plotengine import PlotEngine
import topo.base.topoobject
import topo.base.registry

class TestTopoConsole(unittest.TestCase):
    def setUp(self):
        topo.base.registry.set_active_sim(None)
        self.s = Simulator(register=False)
        self.pe = PlotEngine(self.s)
        self.console = start()
        topo.base.topoobject.min_print_level = topo.base.topoobject.WARNING
        self.s.print_level = topo.base.topoobject.WARNING

    def test_plotengine_dict(self):
        self.assertEqual(self.console.active_simulator(),None)
        self.console.set_active_simulator(self.s)
        self.assertNotEqual(self.console.active_simulator(),None)
        self.assertNotEqual(self.console.active_plotengine(),None)
        self.assertNotEqual(self.console.active_plotengine(),self.pe)
        generated_pe = self.console.active_plotengine()
        self.console.set_active_simulator(None)
        self.assertEqual(self.console.active_simulator(),None)

    def test_do_learning(self):
        run_time = 15.5
        self.console.set_active_simulator(self.s)
        start_time = self.s.time()
        self.console.do_learning(str(run_time))
        end_time = self.s.time()
        self.assertEqual(start_time+run_time,end_time)

    def tearDown(self):
        self.console.quit()

suite = unittest.TestSuite()
#  Uncomment the following line of code, to disable the test if
#  $DISPLAY is undefined.  Used mainly for GUI testing.
suite.requires_display = True
suite.addTest(unittest.makeSuite(TestTopoConsole))
<commit_msg>Fixed broken tests, caused by eliminating topoconsole's copy of the simulator<commit_after>import unittest
from topo.tk import *
from topo.base.simulator import Simulator
from topo.plotting.plotengine import PlotEngine
import topo.base.topoobject
import topo.base.registry

class TestTopoConsole(unittest.TestCase):
    def setUp(self):
        topo.base.registry.set_active_sim(None)
        self.s = Simulator(register=False)
        self.pe = PlotEngine(self.s)
        self.console = start()
        topo.base.topoobject.min_print_level = topo.base.topoobject.WARNING
        self.s.print_level = topo.base.topoobject.WARNING

    def test_plotengine_dict(self):
        self.assertEqual(self.console.active_simulator(),None)
        topo.base.registry.set_active_sim(self.s)
        self.assertNotEqual(self.console.active_simulator(),None)
        self.assertNotEqual(self.console.active_plotengine(),None)
        self.assertNotEqual(self.console.active_plotengine(),self.pe)
        generated_pe = self.console.active_plotengine()
        topo.base.registry.set_active_sim(None)
        self.assertEqual(self.console.active_simulator(),None)

    def test_do_learning(self):
        run_time = 15.5
        topo.base.registry.set_active_sim(self.s)
        start_time = self.s.time()
        self.console.do_learning(str(run_time))
        end_time = self.s.time()
        self.assertEqual(start_time+run_time,end_time)

    def tearDown(self):
        self.console.quit()

suite = unittest.TestSuite()
#  Uncomment the following line of code, to disable the test if
#  $DISPLAY is undefined.  Used mainly for GUI testing.
suite.requires_display = True
suite.addTest(unittest.makeSuite(TestTopoConsole))
<|endoftext|>


<commit_before># encoding: utf-8

###########################################################################################################
#
#
#	Reporter Plugin
#
#	Read the docs:
#	https://github.com/schriftgestalt/GlyphsSDK/tree/master/Python%20Templates/Reporter
#
#
###########################################################################################################

from GlyphsApp.plugins import *

class Symmetry(ReporterPlugin):

	def settings(self):
		self.hFlip = True
		self.vFlip = True

		self.menuName = Glyphs.localize({
			'en': u'Symmetry',
			'de': u'Symmetrie',
			'fr': u'Symmetrie'
		})
		self.generalContextMenus = [
		{
			'name': Glyphs.localize({
				'en': u'Toggle Horizontal Flip',
				'de': u'Schalte horizontale Spiegelung um'}),
			'action': self.toggleHFlip
		},
		{
			'name': Glyphs.localize({
				'en': u'Toggle Vertical Flip',
				'de': u'Schalte vertikale Spiegelung um'}),
			'action': self.toggleVFlip
		},
		]

	def background(self, Layer):
		hScale, vScale = 1.0, 1.0
		xShift, yShift = 0.0, 0.0
		if self.hFlip:
			hScale = -1.0
			xShift = Layer.bounds.origin.x * 2 + Layer.bounds.size.width
		if self.vFlip:
			vScale = -1.0
			yShift = Layer.bounds.origin.y * 2 + Layer.bounds.size.height

		NSColor.colorWithCalibratedRed_green_blue_alpha_( 1.0, 1.0, 0.0, 0.4 ).set()
		bezierPath = Layer.bezierPath
		mirroring = NSAffineTransform.transform()
		mirroring.scaleXBy_yBy_( hScale, vScale )
		mirroring.translateXBy_yBy_( -xShift, -yShift )
		mirroredImage = mirroring.transformBezierPath_( bezierPath )
		mirroredImage.fill()

	def toggleHFlip(self):
		self.hFlip = not self.hFlip

	def toggleVFlip(self):
		self.vFlip = not self.vFlip

<commit_msg>python3<commit_after># encoding: utf-8

###########################################################################################################
#
#
#	Reporter Plugin
#
#	Read the docs:
#	https://github.com/schriftgestalt/GlyphsSDK/tree/master/Python%20Templates/Reporter
#
#
###########################################################################################################

from GlyphsApp.plugins import *

class Symmetry(ReporterPlugin):

	@objc.python_method
	def settings(self):
		self.hFlip = True
		self.vFlip = True

		self.menuName = Glyphs.localize({
			'en': u'Symmetry',
			'de': u'Symmetrie',
			'fr': u'Symmetrie'
		})
		self.generalContextMenus = [
		{
			'name': Glyphs.localize({
				'en': u'Toggle Horizontal Flip',
				'de': u'Schalte horizontale Spiegelung um'}),
			'action': self.toggleHFlip_
		},
		{
			'name': Glyphs.localize({
				'en': u'Toggle Vertical Flip',
				'de': u'Schalte vertikale Spiegelung um'}),
			'action': self.toggleVFlip_
		},
		]

	@objc.python_method
	def background(self, Layer):
		hScale, vScale = 1.0, 1.0
		xShift, yShift = 0.0, 0.0
		if self.hFlip:
			hScale = -1.0
			xShift = Layer.bounds.origin.x * 2 + Layer.bounds.size.width
		if self.vFlip:
			vScale = -1.0
			yShift = Layer.bounds.origin.y * 2 + Layer.bounds.size.height

		NSColor.colorWithCalibratedRed_green_blue_alpha_( 1.0, 1.0, 0.0, 0.4 ).set()
		bezierPath = Layer.bezierPath
		mirroring = NSAffineTransform.transform()
		mirroring.scaleXBy_yBy_( hScale, vScale )
		mirroring.translateXBy_yBy_( -xShift, -yShift )
		mirroredImage = mirroring.transformBezierPath_( bezierPath )
		mirroredImage.fill()

	def toggleHFlip_(self, sender):
		self.hFlip = not self.hFlip

	def toggleVFlip_(self, sender):
		self.vFlip = not self.vFlip

<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the Group Application Model."""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
]

from google.appengine.ext import db

from django.utils.translation import ugettext

import soc.models.document
import soc.models.linkable
import soc.models.user

class GroupApplication(soc.models.linkable.Linkable):
  """Common application questions for all groups.

  Eventually, this will be replaced with a Question/Answer/Quiz/Response
  approach.  At that time, existing OrgApplication entities will be migrated
  (converted) to their new representations in the Datastore.
  """

  #: Required field that will become the name of the Group in the profile,
  #: if the Group Application is accepted.
  #: See also:  soc.models.group.Group.name
  name = db.StringProperty(required=True,
      verbose_name=ugettext('Group Name'))
  name.help_text = ugettext('Complete, formal name of the group.')

  #: Required many:1 relationship indicating the User who is applying on
  #: behalf of the Group.  If the Group Application is accepted, this User
  #: will become the founding User of the Group.
  #: See also:  soc.models.group.Group.founder
  applicant = db.ReferenceProperty(reference_class=soc.models.user.User,
    required=True, collection_name='group_apps',
    verbose_name=ugettext('Applicant'))

  #: Required field indicating the home page URL of the applying Group.
  #: See also:  soc.models.group.Group.home_page
  home_page = db.LinkProperty(required=True,
      verbose_name=ugettext('Home Page URL'))

  #: Required email address used as the "public" contact mechanism for
  #: the Group (as opposed to the applicant.account email address which is
  #: kept secret, revealed only to Developers).
  #: See also:  soc.models.group.Group.email
  email = db.EmailProperty(required=True,
    verbose_name=ugettext('Public Email'))

  #: Required description of the Group.
  description = db.TextProperty(required=True,
      verbose_name=ugettext('Description'))

  why_applying = db.TextProperty(required=True,
    verbose_name=ugettext(
      'Why is your group applying to participate?'
      ' What do you hope to gain by participating?'))

  pub_mailing_list = db.StringProperty(required=False,
    verbose_name=ugettext(
      'What is the main public mailing list for your group?'))
  pub_mailing_list.help_text = ugettext(
    'Mailing list email address, URL to sign-up page, etc.')

  irc_channel = db.StringProperty(required=False,
    verbose_name=ugettext(
      'Where is the main IRC channel for your group?'))
  irc_channel.help_text = ugettext('IRC network and channel.')

  backup_admin = db.ReferenceProperty(reference_class=soc.models.user.User,
    required=True,  collection_name='group_app_backup_admin',
    verbose_name=ugettext(
      'Please select your backup group administrator.'))
  backup_admin.redirect_url = soc.models.user.User.URL_NAME

  member_criteria = db.TextProperty(required=True,
    verbose_name=ugettext(
      'What criteria do you use to select the members of your group?'
      ' Please be as specific as possible.'))
  member_criteria.help_text = ugettext(
    'Members include mentors, admininstrators, and the like.')

  # property containing the status of the application
  # completed means that the application has been processed into a real group
  status = db.StringProperty(required=True,
      choices=['accepted','rejected','ignored','needs review','completed'],
      default='needs review',
      verbose_name=ugettext('Application Status'))

  # timestamp to record the time on which this application has been created
  created_on = db.DateTimeProperty(required=True, auto_now_add=True,
      verbose_name=ugettext('Created on'))

  # timestamp to record the time on which this application has been last modified
  # also changes when the review properties change
  last_modified_on = db.DateTimeProperty(required=True, auto_now=True,
      verbose_name=ugettext('Last modified on'))
<commit_msg>Fixed whitespace in /soc/models/group_app.py<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the Group Application Model."""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
]

from google.appengine.ext import db

from django.utils.translation import ugettext

import soc.models.document
import soc.models.linkable
import soc.models.user

class GroupApplication(soc.models.linkable.Linkable):
  """Common application questions for all groups.

  Eventually, this will be replaced with a Question/Answer/Quiz/Response
  approach.  At that time, existing OrgApplication entities will be migrated
  (converted) to their new representations in the Datastore.
  """

  #: Required field that will become the name of the Group in the profile,
  #: if the Group Application is accepted.
  #: See also:  soc.models.group.Group.name
  name = db.StringProperty(required=True,
      verbose_name=ugettext('Group Name'))
  name.help_text = ugettext('Complete, formal name of the group.')

  #: Required many:1 relationship indicating the User who is applying on
  #: behalf of the Group.  If the Group Application is accepted, this User
  #: will become the founding User of the Group.
  #: See also:  soc.models.group.Group.founder
  applicant = db.ReferenceProperty(reference_class=soc.models.user.User,
    required=True, collection_name='group_apps',
    verbose_name=ugettext('Applicant'))

  #: Required field indicating the home page URL of the applying Group.
  #: See also:  soc.models.group.Group.home_page
  home_page = db.LinkProperty(required=True,
      verbose_name=ugettext('Home Page URL'))

  #: Required email address used as the "public" contact mechanism for
  #: the Group (as opposed to the applicant.account email address which is
  #: kept secret, revealed only to Developers).
  #: See also:  soc.models.group.Group.email
  email = db.EmailProperty(required=True,
    verbose_name=ugettext('Public Email'))

  #: Required description of the Group.
  description = db.TextProperty(required=True,
      verbose_name=ugettext('Description'))

  why_applying = db.TextProperty(required=True,
    verbose_name=ugettext(
      'Why is your group applying to participate?'
      ' What do you hope to gain by participating?'))

  pub_mailing_list = db.StringProperty(required=False,
    verbose_name=ugettext(
      'What is the main public mailing list for your group?'))
  pub_mailing_list.help_text = ugettext(
    'Mailing list email address, URL to sign-up page, etc.')

  irc_channel = db.StringProperty(required=False,
    verbose_name=ugettext(
      'Where is the main IRC channel for your group?'))
  irc_channel.help_text = ugettext('IRC network and channel.')

  backup_admin = db.ReferenceProperty(reference_class=soc.models.user.User,
    required=True,  collection_name='group_app_backup_admin',
    verbose_name=ugettext(
      'Please select your backup group administrator.'))
  backup_admin.redirect_url = soc.models.user.User.URL_NAME

  member_criteria = db.TextProperty(required=True,
    verbose_name=ugettext(
      'What criteria do you use to select the members of your group?'
      ' Please be as specific as possible.'))
  member_criteria.help_text = ugettext(
    'Members include mentors, admininstrators, and the like.')

  # property containing the status of the application
  # completed means that the application has been processed into a real group
  status = db.StringProperty(required=True,
      choices=['accepted','rejected','ignored','needs review','completed'],
      default='needs review',
      verbose_name=ugettext('Application Status'))

  # timestamp to record the time on which this application has been created
  created_on = db.DateTimeProperty(required=True, auto_now_add=True,
      verbose_name=ugettext('Created on'))

  # timestamp to record the time on which this application has been last modified
  # also changes when the review properties change
  last_modified_on = db.DateTimeProperty(required=True, auto_now=True,
      verbose_name=ugettext('Last modified on'))
<|endoftext|>


<commit_before># coding: utf-8

# imports
import os

# django imports
from django import forms
from django.forms.widgets import Input
from django.db.models.fields import Field
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _

# filebrowser imports
from filebrowser.settings import (
    URL_FILEBROWSER_MEDIA, ADMIN_THUMBNAIL, DEBUG, EXTENSIONS
)
from filebrowser.base import FileObject
from filebrowser.functions import url_to_path, _template

class FileBrowseWidget(Input):
    input_type = 'text'

    class Media:
        js = (os.path.join(URL_FILEBROWSER_MEDIA, 'js/AddFileBrowser.js'),)

    def __init__(self, attrs=None):
        self.directory = attrs.get('directory', '')
        self.extensions = attrs.get('extensions', '')
        self.format = attrs.get('format', '')
        super(FileBrowseWidget, self).__init__(attrs)

    def render(self, name, value, attrs=None):
        if value is None:
            value = ""
        final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
        final_attrs['search_icon'] = os.path.join(
            URL_FILEBROWSER_MEDIA, 'img/filebrowser_icon_show.gif'
        )
        final_attrs['directory'] = self.directory
        final_attrs['extensions'] = self.extensions
        final_attrs['format'] = self.format
        final_attrs['ADMIN_THUMBNAIL'] = ADMIN_THUMBNAIL
        final_attrs['DEBUG'] = DEBUG
        if value != "":
            try:
                final_attrs['directory'] = \
                os.path.split(value.path_relative_directory)[0]
            except:
                pass
        return render_to_string(_template() + "custom_field.html", locals())

class FileBrowseFormField(forms.CharField):
    default_error_messages = {
        'extension': _(
            u'Extension %(ext)s is not allowed. Only %(allowed)s is allowed.'),
    }

    def __init__(self, max_length=None, min_length=None,
                 directory=None, extensions=None, format=None,
                 *args, **kwargs):
        self.max_length, self.min_length = max_length, min_length
        self.directory = directory
        self.extensions = extensions
        self.format = format or ''
        self.extensions = extensions or EXTENSIONS.get(format)

        attrs = {
            "directory": self.directory,
            "extensions": self.extensions,
            "format": self.format
        }
        self.widget = FileBrowseWidget(attrs)
        super(FileBrowseFormField, self).__init__(*args, **kwargs)

    def clean(self, value):
        value = super(FileBrowseFormField, self).clean(value)
        if value == '':
            return value
        file_extension = os.path.splitext(value)[1].lower()
        if self.extensions and file_extension not in self.extensions:
            raise forms.ValidationError(
                self.error_messages['extension'] % {'ext': file_extension,
                                                    'allowed': ", ".join(
                                                        self.extensions)})
        return value

class FileBrowseField(Field):
    def __init__(self, *args, **kwargs):
        self.directory = kwargs.pop('directory', '')
        self.extensions = kwargs.pop('extensions', '')
        self.format = kwargs.pop('format', '')
        kwargs['max_length'] = kwargs.get('max_length', 200)
        super(FileBrowseField, self).__init__(*args, **kwargs)

    def from_db_value(self, value, expression, connection, context):
        if not value or isinstance(value, FileObject):
            return value
        return FileObject(url_to_path(value))

    def to_python(self, value):
        if not value or isinstance(value, FileObject):
            return value
        return FileObject(url_to_path(value))

    def get_db_prep_value(self, value, connection, prepared=False):
        if value is None:
            return None
        return str(value)

    # FIXME: recheck or need it
    # @staticmethod
    # def get_manipulator_field_objs():
    #     return [oldforms.TextField]

    def get_internal_type(self):
        return "CharField"

    def formfield(self, **kwargs):
        attrs = {
            "directory": self.directory,
            "extensions": self.extensions,
            "format": self.format
        }
        defaults = {
            'form_class': FileBrowseFormField,
            'widget': FileBrowseWidget(attrs=attrs),
            'directory': self.directory,
            'extensions': self.extensions,
            'format': self.format
        }
        defaults.update(kwargs)
        return super(FileBrowseField, self).formfield(**defaults)
<commit_msg>auto create working directory from field<commit_after># coding: utf-8

# imports
import os

# django imports
from django import forms
from django.forms.widgets import Input
from django.db.models.fields import Field
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _

# filebrowser imports
from filebrowser.settings import (
    URL_FILEBROWSER_MEDIA, ADMIN_THUMBNAIL, DEBUG, EXTENSIONS, MEDIA_ROOT,
    DIRECTORY
)
from filebrowser.base import FileObject
from filebrowser.functions import url_to_path, _template

class FileBrowseWidget(Input):
    input_type = 'text'

    class Media:
        js = (os.path.join(URL_FILEBROWSER_MEDIA, 'js/AddFileBrowser.js'),)

    def __init__(self, attrs=None):
        self.directory = attrs.get('directory', '')
        self.extensions = attrs.get('extensions', '')
        self.format = attrs.get('format', '')
        super(FileBrowseWidget, self).__init__(attrs)
        # if field have a directory - create it
        dir_path = os.path.join(MEDIA_ROOT, DIRECTORY, self.directory)
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    def render(self, name, value, attrs=None):
        if value is None:
            value = ""
        final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
        final_attrs['search_icon'] = os.path.join(
            URL_FILEBROWSER_MEDIA, 'img/filebrowser_icon_show.gif'
        )
        final_attrs['directory'] = self.directory
        final_attrs['extensions'] = self.extensions
        final_attrs['format'] = self.format
        final_attrs['ADMIN_THUMBNAIL'] = ADMIN_THUMBNAIL
        final_attrs['DEBUG'] = DEBUG
        if value != "":
            try:
                final_attrs['directory'] = \
                os.path.split(value.path_relative_directory)[0]
            except:
                pass
        return render_to_string(_template() + "custom_field.html", locals())

class FileBrowseFormField(forms.CharField):
    default_error_messages = {
        'extension': _(
            u'Extension %(ext)s is not allowed. Only %(allowed)s is allowed.'),
    }

    def __init__(self, max_length=None, min_length=None,
                 directory=None, extensions=None, format=None,
                 *args, **kwargs):
        self.max_length, self.min_length = max_length, min_length
        self.directory = directory
        self.extensions = extensions
        self.format = format or ''
        self.extensions = extensions or EXTENSIONS.get(format)

        attrs = {
            "directory": self.directory,
            "extensions": self.extensions,
            "format": self.format
        }
        self.widget = FileBrowseWidget(attrs)
        super(FileBrowseFormField, self).__init__(*args, **kwargs)

    def clean(self, value):
        value = super(FileBrowseFormField, self).clean(value)
        if value == '':
            return value
        file_extension = os.path.splitext(value)[1].lower()
        if self.extensions and file_extension not in self.extensions:
            raise forms.ValidationError(
                self.error_messages['extension'] % {'ext': file_extension,
                                                    'allowed': ", ".join(
                                                        self.extensions)})
        return value

class FileBrowseField(Field):
    def __init__(self, *args, **kwargs):
        self.directory = kwargs.pop('directory', '')
        self.extensions = kwargs.pop('extensions', '')
        self.format = kwargs.pop('format', '')
        kwargs['max_length'] = kwargs.get('max_length', 200)
        super(FileBrowseField, self).__init__(*args, **kwargs)

    def from_db_value(self, value, expression, connection, context):
        if not value or isinstance(value, FileObject):
            return value
        return FileObject(url_to_path(value))

    def to_python(self, value):
        if not value or isinstance(value, FileObject):
            return value
        return FileObject(url_to_path(value))

    def get_db_prep_value(self, value, connection, prepared=False):
        if value is None:
            return None
        return str(value)

    # FIXME: recheck or need it
    # @staticmethod
    # def get_manipulator_field_objs():
    #     return [oldforms.TextField]

    def get_internal_type(self):
        return "CharField"

    def formfield(self, **kwargs):
        attrs = {
            "directory": self.directory,
            "extensions": self.extensions,
            "format": self.format
        }
        defaults = {
            'form_class': FileBrowseFormField,
            'widget': FileBrowseWidget(attrs=attrs),
            'directory': self.directory,
            'extensions': self.extensions,
            'format': self.format
        }
        defaults.update(kwargs)
        return super(FileBrowseField, self).formfield(**defaults)
<|endoftext|>