<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import with_statement
import os
import glob
from shutil import copy
from string import Template
import json

from fabric.api import local, run, sudo, execute, put
from fabric.context_managers import settings, cd, lcd, hide
from fabric.decorators import task, hosts, with_settings
from fabric.utils import abort
from fabric.state import env
from cuisine import package_ensure, dir_ensure, file_link
from git import Repo
from git.remote import Remote
import vagrant

env.repo_host = 'iftp.zalando.net'
env.repo_deb_root = '/data/zalando/iftp.zalando.net/htdocs/repo/apt/'
env.repo_rpm_root = '/data/zalando/iftp.zalando.net/htdocs/repo/centos/'
env.repo_pypi_root = '/data/zalando/iftp.zalando.net/htdocs/simple/'

RPM_RELEASES = ['6']
RPM_COMPONENTS = ['base', 'updates', 'extras']
RPM_ARCHS = ['i386', 'x86_64']

DEB_RELEASES = ['precise', 'trusty']

PACKAGE_FORMAT = {'centos6.5': 'rpm', 'ubuntu12.04': 'deb', 'ubuntu14.04': 'deb'}

### repo commands

# some repo_* commands must run as root, because sudo won't allow access to the GPG keyring

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_rpm_init():
    package_ensure('createrepo')
    dir_ensure('{0}/archive/'.format(env.repo_rpm_root), recursive=True)

    for release in RPM_RELEASES:
        for component in RPM_COMPONENTS:
            for arch in RPM_ARCHS:
                path = '/'.join[env.repo_rpm_root, release, component, arch]
                dir_ensure(path, recursive=True)
                run('createrepo {0}'.format(path))

@hosts(env.repo_host)
@task
def repo_rpm_list(dist='6'):
    with hide('output'):
        output = run('cd {0} && find {1} -type f -name "*rpm"'.format(env.repo_rpm_root, dist))
        for line in output.split('\n'):
            dist, component, arch, package = line.split('/')
            print '{0}: {1}'.format('|'.join([dist, component, arch]), package)

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_rpm_add(package, dist='6', component='base'):

    arch = 'x86_64'
    if any(map(lambda arch: arch in package, ['i386, i586, i686'])):
        arch = 'i386'

    put(package, '{0}/archive/'.format(env.repo_rpm_root))
    package = package.split('/')[-1]
    path = '/'.join([env.repo_rpm_root, dist, component, arch])
    run('cp {0}/archive/{1} {2}'.format(env.repo_rpm_root, package, path))
    run('createrepo {0}'.format(path))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_rpm_del(package, dist='6', component='base'):
    path = '/'.join([env.repo_rpm_root, dist, component])
    run('find {0} -name "*{1}*" -exec mv {{}} {2}/archive/ \;'.format(path, package, env.repo_rpm_root))
    run('createrepo {0}'.format(path))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_deb_init():
    package_ensure('reprepro')
    dir_ensure('{0}/archive/'.format(env.repo_deb_root), recursive=True)
    dir_ensure('{0}/conf/'.format(env.repo_deb_root), recursive=True)

    filename = 'distributions'
    with open('{0}.tmpl'.format(filename), 'r') as tf:
        template = Template(tf.read())

    with open(filename, 'w') as fh:
        for release in DEB_RELEASES:
            fh.write(template.safe_substitute(codename=release, codename_uppercase=release.title()))

    put(filename, '{0}/conf/{1}'.format(env.repo_deb_root, filename))

@hosts(env.repo_host)
@task
def repo_deb_list(dist='precise'):
    sudo('reprepro -b {0} list {1}'.format(env.repo_deb_root, dist))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_deb_add(package, dist='precise'):
    put(package, '{0}/archive/'.format(env.repo_deb_root))
    package = package.split('/')[-1]
    run('reprepro -b {0} includedeb {1} {0}/archive/{2}'.format(env.repo_deb_root, dist, package))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_deb_del(package, dist='precise'):
    run('reprepro -b {0} remove {1} {2}'.format(env.repo_deb_root, dist, package))

### helper commands

@task
def deps_list_debian(package):
    deps = []
    with hide('output', 'running'):
        os.path.isfile(package) or abort('{0} doesn\'t exist.'.format(package))
        deps_string = local('dpkg --field {0} Depends'.format(package), capture=True)
        for dependency in deps_string.split(','):
            dependency = dependency.replace('(', '').replace(')', '').replace(' ', '')
            if not ('>=' in dependency or '<=' in dependency) and dependency.count('=') == 1:
                dependency = dependency.replace('=', '==')
            deps.append(dependency)
    return deps

@task
def build_package(url):

    execute(build_pypi, url)
    execute(prepare_builddir, url)

    path = package_name(url)

    package_dependencies = []
    if os.path.isfile('{0}/package.json'.format(path)):
        with open('{0}/package.json'.format(path), 'r') as fh:
            package_dependencies = json.load(fh).items()
    else:
        package_dependencies = [(t, []) for t in PACKAGE_FORMAT.keys()]

    for target, dependencies in package_dependencies:
        package = None
        package_format = PACKAGE_FORMAT.get(target, 'deb')

        pypi_uri = 'http://pypi.python.org/simple'
        if package_name.startswith('zalando-'):
            pypi_uri = 'http://{0}/simple/'.format(env.repo_host)

        if dependencies:
            dependencies = '--no-auto-depends ' + ' '.join([' -d "{0}"'.format(d) for d in dependencies])

        print 'creating vagrant object with root dir ./{0}'.format(path)
        v = vagrant.Vagrant(root=path)
        print 'running vagrant up for machine {0}'.format(target)
        v.up(vm_name=target)

        with settings(cd('/vagrant'), host_string=v.user_hostname_port(vm_name=target),
                      key_filename=v.keyfile(vm_name=target), disable_known_hosts=True):
            # this is neccesarry because `fpm` looks in a folder equally named like given with the -n option for setup.py to detect the correct version number of the resulting package
            file_link('/vagrant', '/vagrant/{0}'.format(path))
            print 'build {0}.{1} on {2} ({3})'.format(path, package_format, v.user_hostname_port(vm_name=target),
                                                      target)
            messages = sudo('fpm -s python --python-pypi {0} -t {2} {3} --force --name {1} "{1}"'.format(pypi_uri,
                            path, package_format, dependencies))

            for message in messages.split('\n'):
                if 'Created package' in message:
                    package = message.split(':path=>')[1].replace('"', '').replace('}', '')
                    print 'created package "{0}"'.format(package)

        # @TODO detect the correct distribution for uploading into the repos
        if package:
            v.halt(vm_name=target)
            if package_format == 'rpm':
                execute(repo_rpm_add, '{0}/{1}'.format(path, package))
            elif package_format == 'deb':
                execute(repo_deb_add, '{0}/{1}'.format(path, package))
            else:
                print 'no method to add package "{0}" to a repository'.format(package)
        else:
            print 'no package has been created, you may want to inspect the state in the machine:'
            print 'cd {0}/ && vagrant ssh {1}'.format(path, target)

@task
def prepare_builddir(url):
    path = package_name(url)
    execute(git_checkout, url)

    copy('Vagrantfile', path)
    copy('boxes', path)
    for file in glob.glob('provision*sh'):
        copy(file, path)

@task
def build_pypi(url):
    path = package_name(url)
    sha = execute(git_checkout, url)
    sha = sha['<local-only>']

    with lcd(path):
        local('python setup.py -q sdist')
        with settings(host_string=env.repo_host):
            put('dist/*.tar.gz', '{0}/{1}'.format(env.repo_pypi_root, path), use_sudo=True)
        local('find dist -name "*.tar.gz" -exec ln -sf {{}} {0}.tar.gz \;'.format(sha))

def package_name(url):
    ''' get the package name from a git repo url '''

    return url.split('/')[-1].replace('.git', '')

### git commands

@task
def git_checkout(url):
    path = package_name(url)

    if not os.path.isdir(path):
        repo = Repo.clone_from(url, path)
    else:
        repo = Repo(path)
        # ensure that the remote "origin" is set to the correct url
        if 'origin' in [remote.name for remote in repo.remotes]:
            Remote.remove(repo, 'origin')
        Remote.add(repo, 'origin', url)

    repo.remote().pull(refspec='master')
    commit = repo.commit()
    print 'updated to SHA {0}'.format(commit.hexsha)
    return commit.hexsha

<commit_msg>replaced 'deps_list_debian' by more universal task 'package_info'<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import with_statement
import os
import glob
from shutil import copy
from string import Template
import json

from fabric.api import local, run, sudo, execute, put
from fabric.context_managers import settings, cd, lcd, hide
from fabric.decorators import task, hosts, with_settings
from fabric.utils import abort
from fabric.state import env
from cuisine import package_ensure, dir_ensure, file_link
from git import Repo
from git.remote import Remote
import vagrant

env.repo_host = 'iftp.zalando.net'
env.repo_deb_root = '/data/zalando/iftp.zalando.net/htdocs/repo/apt/'
env.repo_rpm_root = '/data/zalando/iftp.zalando.net/htdocs/repo/centos/'
env.repo_pypi_root = '/data/zalando/iftp.zalando.net/htdocs/simple/'

RPM_RELEASES = ['6']
RPM_COMPONENTS = ['base', 'updates', 'extras']
RPM_ARCHS = ['i386', 'x86_64']

DEB_RELEASES = ['precise', 'trusty']

PACKAGE_FORMAT = {'centos6.5': 'rpm', 'ubuntu12.04': 'deb', 'ubuntu14.04': 'deb'}

### repo commands

# some repo_* commands must run as root, because sudo won't allow access to the GPG keyring

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_rpm_init():
    package_ensure('createrepo')
    dir_ensure('{0}/archive/'.format(env.repo_rpm_root), recursive=True)

    for release in RPM_RELEASES:
        for component in RPM_COMPONENTS:
            for arch in RPM_ARCHS:
                path = '/'.join[env.repo_rpm_root, release, component, arch]
                dir_ensure(path, recursive=True)
                run('createrepo {0}'.format(path))

@hosts(env.repo_host)
@task
def repo_rpm_list(dist='6'):
    with hide('output'):
        output = run('cd {0} && find {1} -type f -name "*rpm"'.format(env.repo_rpm_root, dist))
        for line in output.split('\n'):
            dist, component, arch, package = line.split('/')
            print '{0}: {1}'.format('|'.join([dist, component, arch]), package)

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_rpm_add(package, dist='6', component='base'):

    arch = 'x86_64'
    if any(map(lambda arch: arch in package, ['i386, i586, i686'])):
        arch = 'i386'

    put(package, '{0}/archive/'.format(env.repo_rpm_root))
    package = package.split('/')[-1]
    path = '/'.join([env.repo_rpm_root, dist, component, arch])
    run('cp {0}/archive/{1} {2}'.format(env.repo_rpm_root, package, path))
    run('createrepo {0}'.format(path))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_rpm_del(package, dist='6', component='base'):
    path = '/'.join([env.repo_rpm_root, dist, component])
    run('find {0} -name "*{1}*" -exec mv {{}} {2}/archive/ \;'.format(path, package, env.repo_rpm_root))
    run('createrepo {0}'.format(path))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_deb_init():
    package_ensure('reprepro')
    dir_ensure('{0}/archive/'.format(env.repo_deb_root), recursive=True)
    dir_ensure('{0}/conf/'.format(env.repo_deb_root), recursive=True)

    filename = 'distributions'
    with open('{0}.tmpl'.format(filename), 'r') as tf:
        template = Template(tf.read())

    with open(filename, 'w') as fh:
        for release in DEB_RELEASES:
            fh.write(template.safe_substitute(codename=release, codename_uppercase=release.title()))

    put(filename, '{0}/conf/{1}'.format(env.repo_deb_root, filename))

@hosts(env.repo_host)
@task
def repo_deb_list(dist='precise'):
    sudo('reprepro -b {0} list {1}'.format(env.repo_deb_root, dist))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_deb_add(package, dist='precise'):
    put(package, '{0}/archive/'.format(env.repo_deb_root))
    package = package.split('/')[-1]
    run('reprepro -b {0} includedeb {1} {0}/archive/{2}'.format(env.repo_deb_root, dist, package))

@hosts(env.repo_host)
@with_settings(user='root')
@task
def repo_deb_del(package, dist='precise'):
    run('reprepro -b {0} remove {1} {2}'.format(env.repo_deb_root, dist, package))

### helper commands

@task
def package_info(package):
    info = {
        'version': None,
        'description': None,
        'arch': None,
        'dependencies': [],
        'release': None,
    }
    with hide('output', 'running'):
        os.path.isfile(package) or abort('{0} doesn\'t exist.'.format(package))

        if package.endswith('.deb'):
            info['version'] = local('dpkg --field {0} Version'.format(package), capture=True)

            info['description'] = local('dpkg --field {0} Description'.format(package), capture=True)

            info['arch'] = local('dpkg --field {0} Architecture'.format(package), capture=True)

            info['release'] = local('dpkg --field {0} Revision'.format(package), capture=True)

            dependencies = local('dpkg --field {0} Depends'.format(package), capture=True)
            for dependency in dependencies.split(','):
                dependency = dependency.replace('(', '').replace(')', '').replace(' ', '')
                if not ('>=' in dependency or '<=' in dependency) and dependency.count('=') == 1:
                    dependency = dependency.replace('=', '==')
                info['dependencies'].append(dependency)

        if package.endswith('.rpm'):
            info['version'] = local('rpm -qp --dbpath=/tmp --queryformat=%{{VERSION}} {0}'.format(package),
                                    capture=True)

            info['description'] = local('rpm -qp --dbpath=/tmp --queryformat=%{{SUMMARY}} {0}'.format(package),
                                        capture=True)

            info['arch'] = local('rpm -qp --dbpath=/tmp --queryformat=%{{ARCH}} {0}'.format(package), capture=True)

            info['release'] = local('rpm -qp --dbpath=/tmp --queryformat=%{{RELEASE}} {0}'.format(package),
                                    capture=True)

            dependencies = local('rpm -qpR {0}'.format(package), capture=True)
            for dependency in dependencies.split('\n'):
                info['dependencies'].append(dependency.strip())

    print info
    return info

@task
def build_package(url):

    execute(build_pypi, url)
    execute(prepare_builddir, url)

    path = package_name(url)

    package_dependencies = []
    if os.path.isfile('{0}/package.json'.format(path)):
        with open('{0}/package.json'.format(path), 'r') as fh:
            package_dependencies = json.load(fh).items()
    else:
        package_dependencies = [(t, []) for t in PACKAGE_FORMAT.keys()]

    for target, dependencies in package_dependencies:
        package = None
        package_format = PACKAGE_FORMAT.get(target, 'deb')

        pypi_uri = 'http://pypi.python.org/simple'
        if package_name.startswith('zalando-'):
            pypi_uri = 'http://{0}/simple/'.format(env.repo_host)

        if dependencies:
            dependencies = '--no-auto-depends ' + ' '.join([' -d "{0}"'.format(d) for d in dependencies])

        print 'creating vagrant object with root dir ./{0}'.format(path)
        v = vagrant.Vagrant(root=path)
        print 'running vagrant up for machine {0}'.format(target)
        v.up(vm_name=target)

        with settings(cd('/vagrant'), host_string=v.user_hostname_port(vm_name=target),
                      key_filename=v.keyfile(vm_name=target), disable_known_hosts=True):
            # this is neccesarry because `fpm` looks in a folder equally named like given with the -n option for setup.py to detect the correct version number of the resulting package
            file_link('/vagrant', '/vagrant/{0}'.format(path))
            print 'build {0}.{1} on {2} ({3})'.format(path, package_format, v.user_hostname_port(vm_name=target),
                                                      target)
            messages = sudo('fpm -s python --python-pypi {0} -t {2} {3} --force --name {1} "{1}"'.format(pypi_uri,
                            path, package_format, dependencies))

            for message in messages.split('\n'):
                if 'Created package' in message:
                    package = message.split(':path=>')[1].replace('"', '').replace('}', '')
                    print 'created package "{0}"'.format(package)

        # @TODO detect the correct distribution for uploading into the repos
        if package:
            v.halt(vm_name=target)
            if package_format == 'rpm':
                execute(repo_rpm_add, '{0}/{1}'.format(path, package))
            elif package_format == 'deb':
                execute(repo_deb_add, '{0}/{1}'.format(path, package))
            else:
                print 'no method to add package "{0}" to a repository'.format(package)
        else:
            print 'no package has been created, you may want to inspect the state in the machine:'
            print 'cd {0}/ && vagrant ssh {1}'.format(path, target)

@task
def prepare_builddir(url):
    path = package_name(url)
    execute(git_checkout, url)

    copy('Vagrantfile', path)
    copy('boxes', path)
    for file in glob.glob('provision*sh'):
        copy(file, path)

@task
def build_pypi(url):
    path = package_name(url)
    sha = execute(git_checkout, url)
    sha = sha['<local-only>']

    with lcd(path):
        local('python setup.py -q sdist')
        with settings(host_string=env.repo_host):
            put('dist/*.tar.gz', '{0}/{1}'.format(env.repo_pypi_root, path), use_sudo=True)
        local('find dist -name "*.tar.gz" -exec ln -sf {{}} {0}.tar.gz \;'.format(sha))

def package_name(url):
    ''' get the package name from a git repo url '''

    return url.split('/')[-1].replace('.git', '')

### git commands

@task
def git_checkout(url):
    path = package_name(url)

    if not os.path.isdir(path):
        repo = Repo.clone_from(url, path)
    else:
        repo = Repo(path)
        # ensure that the remote "origin" is set to the correct url
        if 'origin' in [remote.name for remote in repo.remotes]:
            Remote.remove(repo, 'origin')
        Remote.add(repo, 'origin', url)

    repo.remote().pull(refspec='master')
    commit = repo.commit()
    print 'updated to SHA {0}'.format(commit.hexsha)
    return commit.hexsha

<|endoftext|>


<commit_before>from __future__ import with_statement
from fabric.api import *
from fabric.contrib.console import confirm
from contextlib import contextmanager

def staging():
    """
    Env parameters for the staging environment.
    """

    env.hosts = ['54.194.210.25']
    env.envname = 'staging'
    env.user = 'ubuntu'
    env.group = 'ubuntu'
    env.key_filename = '~/.ssh/aws_code4sa.pem'
    env.code_dir = '/var/www/odac-ford-housing'
    env.config_dir = 'config_staging'
    env.activate = 'source /var/www/odac-ford-housing/env/bin/activate'
    print("STAGING ENVIRONMENT\n")

@contextmanager
def virtualenv():
    with cd(env.code_dir):
        with prefix(env.activate):
            yield

def configure_redis():

    # upload config file
    put('instance/redis.conf', '/tmp/redis.conf')
    sudo('mv -f /tmp/redis.conf /etc/redis/6379.conf')

    return

def install_redis():
    """
    Install the redis key-value store on port 6379
    http://redis.io/topics/quickstart
    """
    sudo('apt-get install tcl8.5')
    with cd(env['code_dir']):
        sudo('wget http://download.redis.io/redis-stable.tar.gz')
        sudo('tar xvzf redis-stable.tar.gz')
        with cd('redis-stable'):
            sudo('make')
            sudo('make test')
            if confirm("Do you want to continue?"):
                #continue processing
                sudo('cp src/redis-server /usr/local/bin/')
                sudo('cp src/redis-cli /usr/local/bin/')
            with settings(warn_only=True):
                # create dir for config files, data and log
                sudo('mkdir /etc/redis')
                sudo('mkdir /var/redis')
                sudo('touch /var/log/redis_6379.log')
            # init file for handling server restart
            sudo('cp utils/redis_init_script /etc/init.d/redis_6379')
            # copy config file
            sudo('cp redis.conf /etc/redis/6379.conf')
            with settings(warn_only=True):
                # create working directory
                sudo('mkdir /var/redis/6379')

            # ensure redis restarts if the server reboots
            sudo('update-rc.d redis_6379 defaults')

    configure_redis()
    # reboot once, to let redis start up automatically
    sudo('reboot')
    return

def test_redis():

    sudo('redis-cli ping')
    return

def restart_redis():

    with settings(warn_only=True):
        sudo('/etc/init.d/redis_6379 stop')
    sudo('/etc/init.d/redis_6379 start')
    return

def restart():

    with settings(warn_only=True):
        sudo('service nginx restart')
        sudo('service uwsgi restart')
    return

def setup():
    """
    Install dependencies and create an application directory.
    """

    with settings(warn_only=True):
        sudo('service nginx stop')

    # update locale
    sudo('locale-gen en_ZA.UTF-8')

    # install packages
    sudo('apt-get install build-essential python python-dev')
    sudo('apt-get install python-pip')
    sudo('pip install virtualenv')

    # create application directory if it doesn't exist yet
    with settings(warn_only=True):
        if run("test -d %s" % env['code_dir']).failed:
            # create project folder
            sudo('mkdir -p /var/www/odac-ford-housing')
            sudo('mkdir -p /var/www/odac-ford-housing/msg_handler')
            sudo('mkdir /var/www/odac-ford-housing/instance')
        if run("test -d %s/env" % env['code_dir']).failed:
            # create virtualenv
            sudo('virtualenv --no-site-packages %s/env' % env['code_dir'])

    # clear pip's cache
    with settings(warn_only=True):
        sudo('rm -r /tmp/pip-build-root')

    # install the necessary Python packages
    with virtualenv():
        put('requirements/base.txt', '/tmp/base.txt')
        put('requirements/production.txt', '/tmp/production.txt')
        sudo('pip install -r /tmp/production.txt')

    # install nginx
    sudo('apt-get install nginx')
    # restart nginx after reboot
    sudo('update-rc.d nginx defaults')
    sudo('service nginx start')

    # ensure that www-data has access to the application folder
    sudo('chmod -R 770 /var/www/odac-ford-housing')
    sudo('chown -R www-data:www-data /var/www/odac-ford-housing')
    return

def configure():
    """
    Configure uwsgi, Nginx & Flask. Then restart.
    """

    with settings(warn_only=True):
        sudo('stop uwsgi')

    with settings(warn_only=True):
        # disable default site
        sudo('rm /etc/nginx/sites-enabled/default')

    # upload nginx server blocks (virtualhost)
    put(env['config_dir'] + '/nginx.conf', '/tmp/nginx.conf')
    sudo('mv /tmp/nginx.conf /var/www/odac-ford-housing/nginx.conf')

    with settings(warn_only=True):
        sudo('ln -s /var/www/odac-ford-housing/nginx.conf /etc/nginx/conf.d/')

    # upload uwsgi config
    put(env['config_dir'] + '/uwsgi.ini', '/tmp/uwsgi.ini')
    sudo('mv /tmp/uwsgi.ini /var/www/odac-ford-housing/uwsgi.ini')

    # make directory for uwsgi's log
    with settings(warn_only=True):
        sudo('mkdir -p /var/log/uwsgi')

    with settings(warn_only=True):
        sudo('mkdir -p /etc/uwsgi/vassals')

    # upload upstart configuration for uwsgi 'emperor', which spawns uWSGI processes
    put(env['config_dir'] + '/uwsgi.conf', '/tmp/uwsgi.conf')
    sudo('mv /tmp/uwsgi.conf /etc/init/uwsgi.conf')

    with settings(warn_only=True):
        # create symlinks for emperor to find config file
        sudo('ln -s /var/www/odac-ford-housing/uwsgi.ini /etc/uwsgi/vassals')

    # upload flask config
    with settings(warn_only=True):
        sudo('mkdir /var/www/odac-ford-housing/instance')
    put(env['config_dir'] + '/config.py', '/tmp/config.py')
    sudo('mv /tmp/config.py /var/www/odac-ford-housing/instance/config.py')
    put(env['config_dir'] + '/config_private.py', '/tmp/config_private.py')
    sudo('mv /tmp/config_private.py /var/www/odac-ford-housing/instance/config_private.py')

    sudo('chown -R www-data:www-data /var/log/uwsgi')
    sudo('chown -R www-data:www-data /var/www/odac-ford-housing')

    restart()
    return

def deploy():
    """
    Upload our package to the server, unzip it, and restart.
    """

    # create a tarball of our package
    local('tar -czf msg_handler.tar.gz msg_handler/', capture=False)

    # upload the source tarball to the temporary folder on the server
    put('msg_handler.tar.gz', '/tmp/msg_handler.tar.gz')

    with settings(warn_only=True):
        sudo('service nginx stop')

    # enter application directory
    with cd('/var/www/odac-ford-housing'):
        # and unzip new files
        sudo('tar xzf /tmp/msg_handler.tar.gz')

    # now that all is set up, delete the tarball again
    sudo('rm /tmp/msg_handler.tar.gz')
    local('rm msg_handler.tar.gz')

    sudo('touch /var/www/odac-ford-housing/msg_handler/uwsgi.sock')

    # clean out old logfiles
    with settings(warn_only=True):
        sudo('rm /var/www/odac-ford-housing/debug.log*')

    # ensure notification list exists
    sudo('touch /var/www/odac-ford-housing/instance/notification_list.json')

    # ensure user www-data has access to the application folder
    sudo('chown -R www-data:www-data /var/www/odac-ford-housing')
    sudo('chmod -R 775 /var/www/odac-ford-housing')

    # and finally reload the application
    restart()
    return

def upload_db():
    """
    Overwrite the db on the server with a copy of the local db.
    """

    # upload db
    put('instance/ford-housing.db', '/tmp/ford-housing.db')
    sudo('mv /tmp/ford-housing.db /var/www/odac-ford-housing/instance/ford-housing.db')

    # ensure user www-data has access to the application folder
    sudo('chown -R www-data:www-data /var/www/odac-ford-housing')
    sudo('chmod -R 775 /var/www/odac-ford-housing')
    return

def download_db():
    """
    Overwrite the local db with a copy from the server.
    """

    get('/var/www/odac-ford-housing/instance/ford-housing.db', 'instance/ford-housing.db')
    return<commit_msg>Fabfile: configurable application directory.<commit_after>from __future__ import with_statement
from fabric.api import *
from fabric.contrib.console import confirm
from contextlib import contextmanager

def staging():
    """
    Env parameters for the staging environment.
    """

    env.hosts = ['54.194.210.25']
    env.envname = 'staging'
    env.user = 'ubuntu'
    env.group = 'ubuntu'
    env.key_filename = '~/.ssh/aws_code4sa.pem'
    env.code_dir = '/var/www/odac-ford-housing'
    env.config_dir = 'config_staging'
    env.activate = 'source %s/env/bin/activate' % env.code_dir
    print("STAGING ENVIRONMENT\n")

@contextmanager
def virtualenv():
    with cd(env.code_dir):
        with prefix(env.activate):
            yield

def configure_redis():

    # upload config file
    put('instance/redis.conf', '/tmp/redis.conf')
    sudo('mv -f /tmp/redis.conf /etc/redis/6379.conf')

    return

def install_redis():
    """
    Install the redis key-value store on port 6379
    http://redis.io/topics/quickstart
    """
    sudo('apt-get install tcl8.5')
    with cd(env['code_dir']):
        sudo('wget http://download.redis.io/redis-stable.tar.gz')
        sudo('tar xvzf redis-stable.tar.gz')
        with cd('redis-stable'):
            sudo('make')
            sudo('make test')
            if confirm("Do you want to continue?"):
                #continue processing
                sudo('cp src/redis-server /usr/local/bin/')
                sudo('cp src/redis-cli /usr/local/bin/')
            with settings(warn_only=True):
                # create dir for config files, data and log
                sudo('mkdir /etc/redis')
                sudo('mkdir /var/redis')
                sudo('touch /var/log/redis_6379.log')
            # init file for handling server restart
            sudo('cp utils/redis_init_script /etc/init.d/redis_6379')
            # copy config file
            sudo('cp redis.conf /etc/redis/6379.conf')
            with settings(warn_only=True):
                # create working directory
                sudo('mkdir /var/redis/6379')

            # ensure redis restarts if the server reboots
            sudo('update-rc.d redis_6379 defaults')

    configure_redis()
    # reboot once, to let redis start up automatically
    sudo('reboot')
    return

def test_redis():

    sudo('redis-cli ping')
    return

def restart_redis():

    with settings(warn_only=True):
        sudo('/etc/init.d/redis_6379 stop')
    sudo('/etc/init.d/redis_6379 start')
    return

def restart():

    with settings(warn_only=True):
        sudo('service nginx restart')
        sudo('service uwsgi restart')
    return

def setup():
    """
    Install dependencies and create an application directory.
    """

    with settings(warn_only=True):
        sudo('service nginx stop')

    # update locale
    sudo('locale-gen en_ZA.UTF-8')

    # install packages
    sudo('apt-get install build-essential python python-dev')
    sudo('apt-get install python-pip')
    sudo('pip install virtualenv')

    # create application directory if it doesn't exist yet
    with settings(warn_only=True):
        if run("test -d %s" % env['code_dir']).failed:
            # create project folder
            sudo('mkdir -p ' + env.code_dir)
            sudo('mkdir -p %s/msg_handler' % env.code_dir)
            sudo('mkdir %s/instance' % env.code_dir)
        if run("test -d %s/env" % env['code_dir']).failed:
            # create virtualenv
            sudo('virtualenv --no-site-packages %s/env' % env['code_dir'])

    # clear pip's cache
    with settings(warn_only=True):
        sudo('rm -r /tmp/pip-build-root')

    # install the necessary Python packages
    with virtualenv():
        put('requirements/base.txt', '/tmp/base.txt')
        put('requirements/production.txt', '/tmp/production.txt')
        sudo('pip install -r /tmp/production.txt')

    # install nginx
    sudo('apt-get install nginx')
    # restart nginx after reboot
    sudo('update-rc.d nginx defaults')
    sudo('service nginx start')

    # ensure that www-data has access to the application folder
    sudo('chmod -R 770 ' + env.code_dir)
    sudo('chown -R www-data:www-data ' + env.code_dir)
    return

def configure():
    """
    Configure uwsgi, Nginx & Flask. Then restart.
    """

    with settings(warn_only=True):
        sudo('stop uwsgi')

    with settings(warn_only=True):
        # disable default site
        sudo('rm /etc/nginx/sites-enabled/default')

    # upload nginx server blocks (virtualhost)
    put(env['config_dir'] + '/nginx.conf', '/tmp/nginx.conf')
    sudo('mv /tmp/nginx.conf %s/nginx.conf' % env.code_dir)

    with settings(warn_only=True):
        sudo('ln -s %s/nginx.conf /etc/nginx/conf.d/' % env.code_dir)

    # upload uwsgi config
    put(env['config_dir'] + '/uwsgi.ini', '/tmp/uwsgi.ini')
    sudo('mv /tmp/uwsgi.ini %s/uwsgi.ini' % env.code_dir)

    # make directory for uwsgi's log
    with settings(warn_only=True):
        sudo('mkdir -p /var/log/uwsgi')

    with settings(warn_only=True):
        sudo('mkdir -p /etc/uwsgi/vassals')

    # upload upstart configuration for uwsgi 'emperor', which spawns uWSGI processes
    put(env['config_dir'] + '/uwsgi.conf', '/tmp/uwsgi.conf')
    sudo('mv /tmp/uwsgi.conf /etc/init/uwsgi.conf')

    with settings(warn_only=True):
        # create symlinks for emperor to find config file
        sudo('ln -s %s/uwsgi.ini /etc/uwsgi/vassals' % env.code_dir)

    # upload flask config
    with settings(warn_only=True):
        sudo('mkdir %s/instance' % env.code_dir)
    put(env['config_dir'] + '/config.py', '/tmp/config.py')
    sudo('mv /tmp/config.py %s/instance/config.py' % env.code_dir)
    put(env['config_dir'] + '/config_private.py', '/tmp/config_private.py')
    sudo('mv /tmp/config_private.py %s/instance/config_private.py' % env.code_dir)

    sudo('chown -R www-data:www-data /var/log/uwsgi')
    sudo('chown -R www-data:www-data ' + env.code_dir)

    restart()
    return

def deploy():
    """
    Upload our package to the server, unzip it, and restart.
    """

    # create a tarball of our package
    local('tar -czf msg_handler.tar.gz msg_handler/', capture=False)

    # upload the source tarball to the temporary folder on the server
    put('msg_handler.tar.gz', '/tmp/msg_handler.tar.gz')

    with settings(warn_only=True):
        sudo('service nginx stop')

    # enter application directory
    with cd(env.code_dir):
        # and unzip new files
        sudo('tar xzf /tmp/msg_handler.tar.gz')

    # now that all is set up, delete the tarball again
    sudo('rm /tmp/msg_handler.tar.gz')
    local('rm msg_handler.tar.gz')

    sudo('touch %s/msg_handler/uwsgi.sock' % env.code_dir)

    # clean out old logfiles
    with settings(warn_only=True):
        sudo('rm %s/debug.log*' % env.code_dir)

    # ensure notification list exists
    sudo('touch %s/instance/notification_list.json' % env.code_dir)

    # ensure user www-data has access to the application folder
    sudo('chown -R www-data:www-data ' + env.code_dir)
    sudo('chmod -R 775 ' + env.code_dir)

    # and finally reload the application
    restart()
    return

def upload_db():
    """
    Overwrite the db on the server with a copy of the local db.
    """

    # upload db
    put('instance/ford-housing.db', '/tmp/ford-housing.db')
    sudo('mv /tmp/ford-housing.db %s/instance/ford-housing.db' % env.code_dir)

    # ensure user www-data has access to the application folder
    sudo('chown -R www-data:www-data ' + env.code_dir)
    sudo('chmod -R 775 ' + env.code_dir)
    return

def download_db():
    """
    Overwrite the local db with a copy from the server.
    """

    get('%s/instance/ford-housing.db' % env.code_dir, 'instance/ford-housing.db')
    return<|endoftext|>


<commit_before>"""
Deployment file to facilitate releases of pymatgen.
Note that this file is meant to be run from the root directory of the pymatgen
repo.
"""

__author__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "Sep 1, 2014"

import glob
import os
import json
import webbrowser
import requests
import re
import subprocess

from fabric.api import local, lcd
from pymatgen import __version__ as ver

def make_doc():
    with open("CHANGES.rst") as f:
        contents = f.read()

    toks = re.split("\-+", contents)
    n = len(toks[0].split()[-1])
    changes = ("-" * n).join(toks[0:2])

    with open("LATEST_CHANGES.rst", "w") as f:
        f.write(changes)

    with lcd("examples"):
        local("ipython nbconvert --to html *.ipynb")
        local("mv *.html ../docs/_static")
    with lcd("docs"):
        local("cp ../CHANGES.rst change_log.rst")
        local("sphinx-apidoc -d 6 -o . -f ../pymatgen")
        local("rm pymatgen.*.tests.rst")
        for f in glob.glob("docs/*.rst"):
            if f.startswith('docs/pymatgen') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("pymatgen") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        local("make html")
        local("cp _static/* _build/html/_static")

        #This makes sure pymatgen.org works to redirect to the Gihub page
        local("echo \"pymatgen.org\" > _build/html/CNAME")
        #Avoid ths use of jekyll so that _dir works as intended.
        local("touch _build/html/.nojekyll")

    os.remove("LATEST_CHANGES.rst")

def publish():
    local("python setup.py release")

def setver():
    local("sed s/version=.*,/version=\\\"{}\\\",/ setup.py > newsetup"
          .format(ver))
    local("mv newsetup setup.py")

def update_doc():
    make_doc()
    with lcd("docs/_build/html/"):
        local("git add .")
        local("git commit -a -m \"Update dev docs\"")
        local("git push origin gh-pages")

def merge_stable():
    local("git commit -a -m \"v%s release\"" % ver)
    local("git push")
    local("git checkout stable")
    local("git pull")
    local("git merge master")
    local("git push")
    local("git checkout master")

def release_github():
    with open("CHANGES.rst") as f:
        contents = f.read()
    toks = re.split("\-+", contents)
    desc = toks[1].strip()
    payload = {
        "tag_name": "v" + ver,
        "target_commitish": "master",
        "name": "v" + ver,
        "body": desc,
        "draft": False,
        "prerelease": False
    }
    response = requests.post(
        "https://api.github.com/repos/materialsproject/pymatgen/releases",
        data=json.dumps(payload),
        headers={"Authorization": "token " + os.environ["GITHUB_RELEASES_TOKEN"]})
    print response.text

def update_changelog():
    output = subprocess.check_output(["git", "log", "--pretty=format:%s",
        "v%s..HEAD" % ver])
    lines = ["* " + l for l in output.strip().split("\n")]
    with open("CHANGES.rst") as f:
        contents = f.read()
    toks = contents.split("==========")
    toks.insert(-1, "\n\n" + "\n".join(lines))
    print "==========".join(toks)

def log_ver():
    filepath = os.path.join(os.environ["HOME"], "Dropbox", "Public",
                            "pymatgen", ver)
    with open(filepath, "w") as f:
        f.write("Release")

def release(skip_test=False):
    setver()
    if not skip_test:
        local("nosetests")
    publish()
    log_ver()
    update_doc()
    merge_stable()
    release_github()

def open_doc():
    pth = os.path.abspath("docs/_build/html/index.html")
    webbrowser.open("file://" + pth)
<commit_msg>Minor update to fabfile.<commit_after>"""
Deployment file to facilitate releases of pymatgen.
Note that this file is meant to be run from the root directory of the pymatgen
repo.
"""

__author__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "Sep 1, 2014"

import glob
import os
import json
import webbrowser
import requests
import re
import subprocess

from fabric.api import local, lcd
from pymatgen import __version__ as ver

def make_doc():
    with open("CHANGES.rst") as f:
        contents = f.read()

    toks = re.split("\-+", contents)
    n = len(toks[0].split()[-1])
    changes = ("-" * n).join(toks[0:2])

    with open("LATEST_CHANGES.rst", "w") as f:
        f.write(changes)

    with lcd("examples"):
        local("ipython nbconvert --to html *.ipynb")
        local("mv *.html ../docs/_static")
    with lcd("docs"):
        local("cp ../CHANGES.rst change_log.rst")
        local("sphinx-apidoc -d 6 -o . -f ../pymatgen")
        local("rm pymatgen.*.tests.rst")
        for f in glob.glob("docs/*.rst"):
            if f.startswith('docs/pymatgen') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("pymatgen") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        local("make html")
        local("cp _static/* _build/html/_static")

        #This makes sure pymatgen.org works to redirect to the Gihub page
        local("echo \"pymatgen.org\" > _build/html/CNAME")
        #Avoid ths use of jekyll so that _dir works as intended.
        local("touch _build/html/.nojekyll")

    os.remove("LATEST_CHANGES.rst")

def publish():
    local("python setup.py release")

def setver():
    local("sed s/version=.*,/version=\\\"{}\\\",/ setup.py > newsetup"
          .format(ver))
    local("mv newsetup setup.py")

def update_doc():
    make_doc()
    with lcd("docs/_build/html/"):
        local("git add .")
        local("git commit -a -m \"Update dev docs\"")
        local("git push origin gh-pages")

def merge_stable():
    local("git commit -a -m \"v%s release\"" % ver)
    local("git push")
    local("git checkout stable")
    local("git pull")
    local("git merge master")
    local("git push")
    local("git checkout master")

def release_github():
    with open("CHANGES.rst") as f:
        contents = f.read()
    toks = re.split("\-+", contents)
    desc = toks[1].strip()
    payload = {
        "tag_name": "v" + ver,
        "target_commitish": "master",
        "name": "v" + ver,
        "body": desc,
        "draft": False,
        "prerelease": False
    }
    response = requests.post(
        "https://api.github.com/repos/materialsproject/pymatgen/releases",
        data=json.dumps(payload),
        headers={"Authorization": "token " + os.environ["GITHUB_RELEASES_TOKEN"]})
    print response.text

def update_changelog():
    output = subprocess.check_output(["git", "log", "--pretty=format:%s",
        "v%s..HEAD" % ver])
    lines = ["* " + l for l in output.strip().split("\n")]
    with open("CHANGES.rst") as f:
        contents = f.read()
    toks = contents.split("==========")
    toks.insert(-1, "\n\n" + "\n".join(lines))
    with open("CHANGES.rst", "w") as f:
        f.write("==========".join(toks))

def log_ver():
    filepath = os.path.join(os.environ["HOME"], "Dropbox", "Public",
                            "pymatgen", ver)
    with open(filepath, "w") as f:
        f.write("Release")

def release(skip_test=False):
    setver()
    if not skip_test:
        local("nosetests")
    publish()
    log_ver()
    update_doc()
    merge_stable()
    release_github()

def open_doc():
    pth = os.path.abspath("docs/_build/html/index.html")
    webbrowser.open("file://" + pth)
<|endoftext|>


<commit_before>"""
Starter fabfile for deploying the medfuturo project.

Change all the things marked CHANGEME. Other things can be left at their
defaults if you are happy with the default layout.
"""

import posixpath

from fabric.api import run, local, env, settings, cd, task
from fabric.contrib.files import exists
from fabric.operations import _prefix_commands, _prefix_env_vars
#from fabric.decorators import runs_once
#from fabric.context_managers import cd, lcd, settings, hide

# CHANGEME
env.hosts = ['user@medfuturo.example.com']
env.code_dir = '/srv/www/medfuturo'
env.project_dir = '/srv/www/medfuturo/medfuturo'
env.static_root = '/srv/www/medfuturo/static/'
env.virtualenv = '/srv/www/medfuturo/.virtualenv'
env.code_repo = 'git@github.com:raonyguimaraes/medfuturo.git'
env.django_settings_module = 'medfuturo.settings'

env.user  = 'raony'
env.hosts = ['webfaction']

# Python version
PYTHON_BIN = "python2.7"
PYTHON_PREFIX = ""  # e.g. /usr/local  Use "" for automatic
PYTHON_FULL_PATH = "%s/bin/%s" % (PYTHON_PREFIX, PYTHON_BIN) if PYTHON_PREFIX else PYTHON_BIN

# Set to true if you can restart your webserver (via wsgi.py), false to stop/start your webserver
# CHANGEME
DJANGO_SERVER_RESTART = False

def virtualenv(venv_dir):
    """
    Context manager that establishes a virtualenv to use.
    """
    return settings(venv=venv_dir)

def run_venv(command, **kwargs):
    """
    Runs a command in a virtualenv (which has been specified using
    the virtualenv context manager
    """
    run("source %s/bin/activate" % env.virtualenv + " && " + command, **kwargs)

def install_dependencies():
    ensure_virtualenv()
    with virtualenv(env.virtualenv):
        with cd(env.code_dir):
            run_venv("pip install -r requirements/production.txt")

def ensure_virtualenv():
    if exists(env.virtualenv):
        return

    with cd(env.code_dir):
        run("virtualenv --no-site-packages --python=%s %s" %
            (PYTHON_BIN, env.virtualenv))
        run("echo %s > %s/lib/%s/site-packages/projectsource.pth" %
            (env.project_dir, env.virtualenv, PYTHON_BIN))

def ensure_src_dir():
    if not exists(env.code_dir):
        run("mkdir -p %s" % env.code_dir)
    with cd(env.code_dir):
        if not exists(posixpath.join(env.code_dir, '.git')):
            run('git clone %s .' % (env.code_repo))

def push_sources():
    """
    Push source code to server
    """
    ensure_src_dir()
    local('git push origin master')
    with cd(env.code_dir):
        run('git pull origin master')

@task
def run_tests():
    """ Runs the Django test suite as is.  """
    local("./manage.py test")

@task
def version():
    """ Show last commit to the deployed repo. """
    with cd(env.code_dir):
        run('git log -1')

@task
def uname():
    """ Prints information about the host. """
    run("uname -a")

@task
def webserver_stop():
    """
    Stop the webserver that is running the Django instance
    """
    run("service apache2 stop")

@task
def webserver_start():
    """
    Starts the webserver that is running the Django instance
    """
    run("service apache2 start")

@task
def webserver_restart():
    """
    Restarts the webserver that is running the Django instance
    """
    if DJANGO_SERVER_RESTART:
        with cd(env.code_dir):
            run("touch %s/wsgi.py" % env.project_dir)
    else:
        with settings(warn_only=True):
            webserver_stop()
        webserver_start()

def restart():
    """ Restart the wsgi process """
    with cd(env.code_dir):
        run("touch %s/medfuturo/wsgi.py" % env.code_dir)

def build_static():
    assert env.static_root.strip() != '' and env.static_root.strip() != '/'
    with virtualenv(env.virtualenv):
        with cd(env.code_dir):
            run_venv("./manage.py collectstatic -v 0 --clear --noinput")

    run("chmod -R ugo+r %s" % env.static_root)

@task
def first_deployment_mode():
    """
    Use before first deployment to switch on fake south migrations.
    """
    env.initial_deploy = True

@task
def update_database(app=None):
    """
    Update the database (run the migrations)
    Usage: fab update_database:app_name
    """
    with virtualenv(env.virtualenv):
        with cd(env.code_dir):
            if getattr(env, 'initial_deploy', False):
                run_venv("./manage.py syncdb --all")
                run_venv("./manage.py migrate --fake --noinput")
            else:
                run_venv("./manage.py syncdb --noinput")
                if app:
                    run_venv("./manage.py migrate %s --noinput" % app)
                else:
                    run_venv("./manage.py migrate --noinput")

@task
def sshagent_run(cmd):
    """
    Helper function.
    Runs a command with SSH agent forwarding enabled.

    Note:: Fabric (and paramiko) can't forward your SSH agent.
    This helper uses your system's ssh to do so.
    """
    # Handle context manager modifications
    wrapped_cmd = _prefix_commands(_prefix_env_vars(cmd), 'remote')
    try:
        host, port = env.host_string.split(':')
        return local(
            "ssh -p %s -A %s@%s '%s'" % (port, env.user, host, wrapped_cmd)
        )
    except ValueError:
        return local(
            "ssh -A %s@%s '%s'" % (env.user, env.host_string, wrapped_cmd)
        )

@task
def deploy():
    """
    Deploy the project.
    """
    with settings(warn_only=True):
        webserver_stop()
    push_sources()
    install_dependencies()
    update_database()
    build_static()
    webserver_start()

@task
def deploy_medfuturo():
    print 'Hello World'
    local("git add .")
    local("""git commit -m 'changes' """)

    local("git push")

    with cd('/home/raony/webapps/medfuturo/medfuturo/'):
        run('git pull')
        run('/home/raony/webapps/medfuturo/apache2/bin/restart')<commit_msg>changes<commit_after>"""
Starter fabfile for deploying the medfuturo project.

Change all the things marked CHANGEME. Other things can be left at their
defaults if you are happy with the default layout.
"""

import posixpath

from fabric.api import run, local, env, settings, cd, task
from fabric.contrib.files import exists
from fabric.operations import _prefix_commands, _prefix_env_vars
#from fabric.decorators import runs_once
#from fabric.context_managers import cd, lcd, settings, hide

# CHANGEME
env.hosts = ['user@medfuturo.example.com']
env.code_dir = '/srv/www/medfuturo'
env.project_dir = '/srv/www/medfuturo/medfuturo'
env.static_root = '/srv/www/medfuturo/static/'
env.virtualenv = '/srv/www/medfuturo/.virtualenv'
env.code_repo = 'git@github.com:raonyguimaraes/medfuturo.git'
env.django_settings_module = 'medfuturo.settings'

env.user  = 'raony'
env.hosts = ['webfaction']

# Python version
PYTHON_BIN = "python2.7"
PYTHON_PREFIX = ""  # e.g. /usr/local  Use "" for automatic
PYTHON_FULL_PATH = "%s/bin/%s" % (PYTHON_PREFIX, PYTHON_BIN) if PYTHON_PREFIX else PYTHON_BIN

# Set to true if you can restart your webserver (via wsgi.py), false to stop/start your webserver
# CHANGEME
DJANGO_SERVER_RESTART = False

def virtualenv(venv_dir):
    """
    Context manager that establishes a virtualenv to use.
    """
    return settings(venv=venv_dir)

def run_venv(command, **kwargs):
    """
    Runs a command in a virtualenv (which has been specified using
    the virtualenv context manager
    """
    run("source %s/bin/activate" % env.virtualenv + " && " + command, **kwargs)

def install_dependencies():
    ensure_virtualenv()
    with virtualenv(env.virtualenv):
        with cd(env.code_dir):
            run_venv("pip install -r requirements/production.txt")

def ensure_virtualenv():
    if exists(env.virtualenv):
        return

    with cd(env.code_dir):
        run("virtualenv --no-site-packages --python=%s %s" %
            (PYTHON_BIN, env.virtualenv))
        run("echo %s > %s/lib/%s/site-packages/projectsource.pth" %
            (env.project_dir, env.virtualenv, PYTHON_BIN))

def ensure_src_dir():
    if not exists(env.code_dir):
        run("mkdir -p %s" % env.code_dir)
    with cd(env.code_dir):
        if not exists(posixpath.join(env.code_dir, '.git')):
            run('git clone %s .' % (env.code_repo))

def push_sources():
    """
    Push source code to server
    """
    ensure_src_dir()
    local('git push origin master')
    with cd(env.code_dir):
        run('git pull origin master')

@task
def run_tests():
    """ Runs the Django test suite as is.  """
    local("./manage.py test")

@task
def version():
    """ Show last commit to the deployed repo. """
    with cd(env.code_dir):
        run('git log -1')

@task
def uname():
    """ Prints information about the host. """
    run("uname -a")

@task
def webserver_stop():
    """
    Stop the webserver that is running the Django instance
    """
    run("service apache2 stop")

@task
def webserver_start():
    """
    Starts the webserver that is running the Django instance
    """
    run("service apache2 start")

@task
def webserver_restart():
    """
    Restarts the webserver that is running the Django instance
    """
    if DJANGO_SERVER_RESTART:
        with cd(env.code_dir):
            run("touch %s/wsgi.py" % env.project_dir)
    else:
        with settings(warn_only=True):
            webserver_stop()
        webserver_start()

def restart():
    """ Restart the wsgi process """
    with cd(env.code_dir):
        run("touch %s/medfuturo/wsgi.py" % env.code_dir)

def build_static():
    assert env.static_root.strip() != '' and env.static_root.strip() != '/'
    with virtualenv(env.virtualenv):
        with cd(env.code_dir):
            run_venv("./manage.py collectstatic -v 0 --clear --noinput")

    run("chmod -R ugo+r %s" % env.static_root)

@task
def first_deployment_mode():
    """
    Use before first deployment to switch on fake south migrations.
    """
    env.initial_deploy = True

@task
def update_database(app=None):
    """
    Update the database (run the migrations)
    Usage: fab update_database:app_name
    """
    with virtualenv(env.virtualenv):
        with cd(env.code_dir):
            if getattr(env, 'initial_deploy', False):
                run_venv("./manage.py syncdb --all")
                run_venv("./manage.py migrate --fake --noinput")
            else:
                run_venv("./manage.py syncdb --noinput")
                if app:
                    run_venv("./manage.py migrate %s --noinput" % app)
                else:
                    run_venv("./manage.py migrate --noinput")

@task
def sshagent_run(cmd):
    """
    Helper function.
    Runs a command with SSH agent forwarding enabled.

    Note:: Fabric (and paramiko) can't forward your SSH agent.
    This helper uses your system's ssh to do so.
    """
    # Handle context manager modifications
    wrapped_cmd = _prefix_commands(_prefix_env_vars(cmd), 'remote')
    try:
        host, port = env.host_string.split(':')
        return local(
            "ssh -p %s -A %s@%s '%s'" % (port, env.user, host, wrapped_cmd)
        )
    except ValueError:
        return local(
            "ssh -A %s@%s '%s'" % (env.user, env.host_string, wrapped_cmd)
        )

@task
def deploy():
    """
    Deploy the project.
    """
    with settings(warn_only=True):
        webserver_stop()
    push_sources()
    install_dependencies()
    update_database()
    build_static()
    webserver_start()

@task
def deploy_medfuturo():
    print 'Hello World'
    local("git add .")
    local("""git commit -m "changes" """)

    local("git push")

    with cd('/home/raony/webapps/medfuturo/medfuturo/'):
        run('git pull')
        run('/home/raony/webapps/medfuturo/apache2/bin/restart')<|endoftext|>


<commit_before>from __future__ import with_statement

from fabric.api import env
from fabric.api import hide
from fabric.api import run
from fabric.api import local
from fabric.api import sudo
from fabric.api import task
from fabric.api import abort
from fabric.context_managers import cd

import os
import sys

HOME_DIR = "cloudlet/gabriel/"
CONTROL_DIR = "./src/control"
UCOMM_DIR = "./src/ucomm"
APP_COMM_DIR = "./src/app/common"
COMMON_FILES = ["./src/control/__init__.py",
                "./src/control/protocol.py",
                "./src/control/config.py",
                "./src/control/upnp_client.py",
                "./src/control/log.py",
                "./src/control/lib/gabriel_upnp_client.jar",
                ]
BUILD_DIR = "./dist"

def check_support():
    # check ubuntu
    if run("lsb_release -irc | grep Ubuntu > /dev/null").failed:
        abort("This installation script is designed for Ubuntu distribution")

@task
def localhost():
    env.run = local
    env.warn_only = True
    env.hosts = ['localhost']

@task
def install():
    check_support()

    # install dependent package
    sudo("apt-get update")
    if sudo("apt-get install default-jre python-pip " +
            "pssh python-psutil").failed:
        abort("Failed to install libraries")
    if sudo("sudo pip install flask==0.9 flask-restful==0.2.1 six==1.1.0").failed:
        abort("Failed to install python libraries")

    sys.stdout.write("[SUCCESS] Finish installing Gabriel\n")

def _tar_dir(tarballname, dirname):
    if run("tar -cvf %s %s" % (tarballname, dirname)).failed:
        msg = "Failed to compress directories:\n"
        msg += "%s\n" % (CONTROL_DIR)
        abort(msg)

def _tar_files(tarballname, filelist, exclude_list=list(), comp_tool="tar"):
    global BUILD_DIR

    if os.path.exists(BUILD_DIR) is False:
        os.mkdir(BUILD_DIR)

    filelist_str = ' '.join(filelist)
    exclude_str = ''
    for item in exclude_list:
        exclude_str += '--exclude=%s ' % item
    tarballpath = os.path.join(BUILD_DIR, tarballname)

    command = None
    if comp_tool == "tar":
        command = "tar cvfz %s %s %s" % (tarballpath, filelist_str, exclude_str)
    elif comp_tool == "zip":
        command = "zip -r %s %s %s" % (tarballpath, filelist_str, exclude_str)
    else:
        msg = "Unsupported compression tool name: %s" % comp_tool
        abort(msg)

    if run(command).failed:
        msg = "Failed to compress directories:\n"
        msg += "%s\n" % (filelist_str)
        abort(msg)

@task
def packaging():
    global HOME_DIR
    global CONTROL_DIR
    global UCOMM_DIR
    global APP_COMM_DIR

    packaging_control()
    packaging_ucomm()
    packaging_appcomm()

@task
def packaging_control():
    global HOME_DIR
    global CONTROL_DIR

    tarfile_name = "gabriel-control.tar.gz"
    with cd(HOME_DIR):
        _tar_dir(tarfile_name, CONTROL_DIR)

@task
def packaging_ucomm():
    global HOME_DIR
    global UCOMM_DIR
    global COMMON_FILES

    tarballname = "gabriel-ucomm.tar.gz"
    with cd(HOME_DIR):
        filelist = COMMON_FILES + [UCOMM_DIR]
        _tar_files(tarballname, filelist)

@task
def packaging_appcomm():
    global HOME_DIR
    global APP_COMM_DIR
    global COMMON_FILES

    tarballname = "gabriel-appcomm.tar.gz"
    with cd(HOME_DIR):
        filelist = COMMON_FILES + [APP_COMM_DIR]
        _tar_files(tarballname, filelist)

@task
def packaging_apps():
    global HOME_DIR
    global APP_COMM_DIR
    global COMMON_FILES

    with cd(HOME_DIR):
        '''
        # dummy
        filelist = COMMON_FILES + [APP_COMM_DIR] + ["./src/app/http_display/"]
        _tar_files("gabriel-dummy.tar.gz", filelist)
        '''
        # moped
        filelist = COMMON_FILES + [APP_COMM_DIR] + ["./src/app/object_moped/"]
        _tar_files("gabriel-moped.tar.gz", filelist)

        # OCR (open source)
        filelist = COMMON_FILES + [APP_COMM_DIR] + ["./src/app/ocr/"]
        exclude_list = ["./src/app/ocr/results", "./src/app/ocr/glass-client"]
        _tar_files("gabriel-ocr.tar.gz", filelist, exclude_list=exclude_list)

        # face recognition (Windows)
        filelist = ["./src/app/face_recognition/dist"]
        _tar_files("gabriel-face.zip", filelist, comp_tool="zip")

        # AR (Windows)
        filelist = ["./src/app/mar/dist"]
        _tar_files("gabriel-mar.zip", filelist, comp_tool="zip")
<commit_msg>use _tar_files method for directory as well.<commit_after>from __future__ import with_statement

from fabric.api import env
from fabric.api import hide
from fabric.api import run
from fabric.api import local
from fabric.api import sudo
from fabric.api import task
from fabric.api import abort
from fabric.context_managers import cd

import os
import sys

HOME_DIR = "cloudlet/gabriel/"
CONTROL_DIR = "./src/control"
UCOMM_DIR = "./src/ucomm"
APP_COMM_DIR = "./src/app/common"
COMMON_FILES = ["./src/control/__init__.py",
                "./src/control/protocol.py",
                "./src/control/config.py",
                "./src/control/upnp_client.py",
                "./src/control/log.py",
                "./src/control/lib/gabriel_upnp_client.jar",
                ]
BUILD_DIR = "./dist"

def check_support():
    # check ubuntu
    if run("lsb_release -irc | grep Ubuntu > /dev/null").failed:
        abort("This installation script is designed for Ubuntu distribution")

@task
def localhost():
    env.run = local
    env.warn_only = True
    env.hosts = ['localhost']

@task
def install():
    check_support()

    # install dependent package
    sudo("apt-get update")
    if sudo("apt-get install default-jre python-pip " +
            "pssh python-psutil").failed:
        abort("Failed to install libraries")
    if sudo("sudo pip install flask==0.9 flask-restful==0.2.1 six==1.1.0").failed:
        abort("Failed to install python libraries")

    sys.stdout.write("[SUCCESS] Finish installing Gabriel\n")

def _tar_files(tarballname, filelist, exclude_list=list(), comp_tool="tar"):
    global BUILD_DIR

    if os.path.exists(BUILD_DIR) is False:
        os.mkdir(BUILD_DIR)

    filelist_str = ' '.join(filelist)
    exclude_str = ''
    for item in exclude_list:
        exclude_str += '--exclude=%s ' % item
    tarballpath = os.path.join(BUILD_DIR, tarballname)

    command = None
    if comp_tool == "tar":
        command = "tar cvfz %s %s %s" % (tarballpath, filelist_str, exclude_str)
    elif comp_tool == "zip":
        command = "zip -r %s %s %s" % (tarballpath, filelist_str, exclude_str)
    else:
        msg = "Unsupported compression tool name: %s" % comp_tool
        abort(msg)

    if run(command).failed:
        msg = "Failed to compress directories:\n"
        msg += "%s\n" % (filelist_str)
        abort(msg)

@task
def packaging():
    global HOME_DIR
    global CONTROL_DIR
    global UCOMM_DIR
    global APP_COMM_DIR

    packaging_control()
    packaging_ucomm()
    packaging_apps()

@task
def packaging_control():
    global HOME_DIR
    global CONTROL_DIR

    tarfile_name = "gabriel-control.tar.gz"
    with cd(HOME_DIR):
        filelist = [CONTROL_DIR]
        _tar_files(tarfile_name, filelist)

@task
def packaging_ucomm():
    global HOME_DIR
    global UCOMM_DIR
    global COMMON_FILES

    tarballname = "gabriel-ucomm.tar.gz"
    with cd(HOME_DIR):
        filelist = COMMON_FILES + [UCOMM_DIR]
        _tar_files(tarballname, filelist)

@task
def packaging_appcomm():
    global HOME_DIR
    global APP_COMM_DIR
    global COMMON_FILES

    tarballname = "gabriel-appcomm.tar.gz"
    with cd(HOME_DIR):
        filelist = COMMON_FILES + [APP_COMM_DIR]
        _tar_files(tarballname, filelist)

@task
def packaging_apps():
    global HOME_DIR
    global APP_COMM_DIR
    global COMMON_FILES

    with cd(HOME_DIR):
        '''
        # dummy
        filelist = COMMON_FILES + [APP_COMM_DIR] + ["./src/app/http_display/"]
        _tar_files("gabriel-dummy.tar.gz", filelist)
        '''
        # moped
        filelist = COMMON_FILES + [APP_COMM_DIR] + ["./src/app/object_moped/"]
        _tar_files("gabriel-moped.tar.gz", filelist)

        # OCR (open source)
        filelist = COMMON_FILES + [APP_COMM_DIR] + ["./src/app/ocr/"]
        exclude_list = ["./src/app/ocr/results", "./src/app/ocr/glass-client"]
        _tar_files("gabriel-ocr.tar.gz", filelist, exclude_list=exclude_list)

        # face recognition (Windows)
        filelist = ["./src/app/face_recognition/dist"]
        _tar_files("gabriel-face.zip", filelist, comp_tool="zip")

        # AR (Windows)
        filelist = ["./src/app/mar/dist"]
        _tar_files("gabriel-mar.zip", filelist, comp_tool="zip")
<|endoftext|>


<commit_before>"""Model independent definitions for running models."""
# Copyright 2016 Andrew Dawson
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from abc import ABCMeta, abstractmethod
from collections import namedtuple
from datetime import timedelta
import glob
import os
from os.path import join as pjoin
from tempfile import mkdtemp

import xray as xr

from .exceptions import TileInitializationError, TileRunError
from .util import get_logger

#: The result of a single cell simulation.
CellResult = namedtuple('CellResult', ('cell', 'outputs'))

#: The result of a whole tile of simulations.
TileResult = namedtuple('TileResult', ('id', 'cell_results'))

class TileRunner(metaclass=ABCMeta):

    def __init__(self, config, tile, tile_in_memory=True):
        """
        Construct a tile runner.

        **Arguments:**

        * config: `scmtiles.config.SCMTilesConfig`
            A configuration object defining the parameters for the tile.

        * tile: `scmtiles.grid_manager.Tile`
            A tile defining the work required of the runner.

        **Optional argument:**

        * tile_in_memory
            If `True` then the data for the tile will be loaded from
            file into memory at initialization time. If `False` then
            data from the file will be read as needed from the file.
            Defaults to `True`.

        """
        self.config = config
        self.tile = tile
        self._tile_data = self._load_tile_data(in_memory=tile_in_memory)

    def _load_tile_data(self, in_memory=True):
        # Add new configuration items forcing_step_seconds and
        # forcing_num_steps which can be used to gather the correct input times
        forcing_length = self.config.forcing_num_steps * \
            self.config.forcing_step_seconds
        forcing_leads = range(0, forcing_length + 1,
                              self.config.forcing_step_seconds)
        forcing_times = [self.config.start_time + timedelta(seconds=n)
                         for n in forcing_leads]
        input_file_names = [self.config.input_file_pattern.format(
                                tile=self.tile, start_time=t)
                            for t in forcing_times]
        input_file_paths = [pjoin(self.config.input_directory, f)
                            for f in input_file_names]
        # Define the tile selector.
        if self.tile.type == 'linear':
            selector = {'grid': self.tile.selector}
        else:
            selector = {self.config.xname: self.tile.xselector,
                        self.config.yname: self.tile.yselector}
        try:
            with xr.open_mfdataset(input_file_paths) as ds:
                if self.tile.type == 'linear':
                    # Reconfigure the grid for linear tiles.
                    ds = ds.stack(grid=(self.config.yname, self.config.xname))
                tile_ds = ds.isel(**selector)
                if in_memory:
                    tile_ds.load()
        except RuntimeError:
            msg = 'Failed to open input file "{}".'
            raise TileInitializationError(msg.format(input_file_path))
        return tile_ds

    def run(self):
        """Start the tile's runs in serial."""
        log_file_name = 'run.{:03d}.{}.log'.format(
            self.tile.id, self.config.start_time.strftime('%Y%m%d%H%M%S'))
        log_file_path = pjoin(self.config.work_directory, log_file_name)
        # Create the work directory if it doesn't exist.
        try:
            os.makedirs(self.config.work_directory, exist_ok=True)
        except PermissionError as e:
            msg = 'Cannot create work directory "{}", permission denied.'
            raise TileRunError(msg.format(run_directory))
        with open(log_file_path, 'w') as lf:
            # Write a header to the run log file.
            header = 'Tile: {!s}`n'.format(self.tile)
            lf.write(header)
            log = get_logger(lf)
            tile_result = TileResult(id=self.tile.id, cell_results=[])
            for cell in self.tile.cells():
                cell_result = self.run_cell(cell, logger=log)
                tile_result.cell_results.append(cell_result)
            log('Finished tile #{:03d}'.format(self.tile.id))
        return tile_result

    @abstractmethod
    def run_cell(self, cell, logger=print):
        """
        Run an individual cell of the tile.

        **Argument:**

        * cell
            A `Cell` instance des cribing the cell to be run.

        **Keyword argument:**

        * logger
            A callable logging function. Defaults to `print`.

        **Returns:**

        * cell_result
            A `CellResult` object describing the result of the run.

        .. note:: This method must be implemented by the derived class.

        """
        raise NotImplementedError('run_cell() must be defined.')

    def get_cell(self, cell):
        """
        Retrieve the `xarray.Dataset` corresponding to a given cell
        belonging to the tile.

        **Arguments:**

        * cell: `scmtiles.grid_manager.Cell`
            The cell to extract from the tile.

        **Returns:**

        * cell_ds: `xarray.Dataset`
            The dataset corresponding to the required cell.

        """
        if self.tile.type == 'linear':
            selector = {'grid': cell.x}
        else:
            selector = {self.config.xname: cell.x, self.config.yname: cell.y}
        cell_ds = self._tile_data.isel(**selector)
        if self.tile.type == 'linear':
            # Convert the grid coordinate back to latitude and longitude.
            # NB: cell_ds.unstack('grid') does not appear to work.
            lat, lon = cell_ds['grid'].data
            cell_ds.coords.update({self.config.yname: lat,
                                   self.config.xname: lon})
            cell_ds = cell_ds.drop('grid')
        return cell_ds

    def create_run_directory(self):
        """
        Create a temporary run directory.

        **Returns:**

        * run_directory
            The path to the created directory.

        """
        try:
            run_directory = mkdtemp(dir=self.config.work_directory,
                                    prefix='run.')
        except PermissionError:
            msg = 'Cannot create run directory "{}", permission denied.'
            raise TileRunError(msg.format(run_directory))
        return run_directory

    def link_template(self, run_directory):
        """
        Link all filesin the template directory to the run directory.

        **Argument:**

        * run_directory
            Path to the run directory in which links will be created.

        **Returns:**

        * linked_files
            A list of 2-tuples containing the full paths to the source
            and target of each linked file.

        """
        linked_files = []
        template_pattern = '{}/*'.format(self.config.template_directory)
        for source in glob.glob(template_pattern):
            target = pjoin(run_directory, os.path.basename(source))
            try:
                os.symlink(source, target)
            except PermissionError:
                msg = 'Cannot create links in "{}", permission denied.'
                raise TileRunError(msg.format(run_directory))
            else:
                linked_files.append((source, target))
        return linked_files
<commit_msg>Correct error handling for netcdf input.<commit_after>"""Model independent definitions for running models."""
# Copyright 2016 Andrew Dawson
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from abc import ABCMeta, abstractmethod
from collections import namedtuple
from datetime import timedelta
import glob
import os
from os.path import join as pjoin
from tempfile import mkdtemp

import xray as xr

from .exceptions import TileInitializationError, TileRunError
from .util import get_logger

#: The result of a single cell simulation.
CellResult = namedtuple('CellResult', ('cell', 'outputs'))

#: The result of a whole tile of simulations.
TileResult = namedtuple('TileResult', ('id', 'cell_results'))

class TileRunner(metaclass=ABCMeta):

    def __init__(self, config, tile, tile_in_memory=True):
        """
        Construct a tile runner.

        **Arguments:**

        * config: `scmtiles.config.SCMTilesConfig`
            A configuration object defining the parameters for the tile.

        * tile: `scmtiles.grid_manager.Tile`
            A tile defining the work required of the runner.

        **Optional argument:**

        * tile_in_memory
            If `True` then the data for the tile will be loaded from
            file into memory at initialization time. If `False` then
            data from the file will be read as needed from the file.
            Defaults to `True`.

        """
        self.config = config
        self.tile = tile
        self._tile_data = self._load_tile_data(in_memory=tile_in_memory)

    def _load_tile_data(self, in_memory=True):
        # Add new configuration items forcing_step_seconds and
        # forcing_num_steps which can be used to gather the correct input times
        forcing_length = self.config.forcing_num_steps * \
            self.config.forcing_step_seconds
        forcing_leads = range(0, forcing_length + 1,
                              self.config.forcing_step_seconds)
        forcing_times = [self.config.start_time + timedelta(seconds=n)
                         for n in forcing_leads]
        input_file_names = [self.config.input_file_pattern.format(
                                tile=self.tile, start_time=t)
                            for t in forcing_times]
        input_file_paths = [pjoin(self.config.input_directory, f)
                            for f in input_file_names]
        # Define the tile selector.
        if self.tile.type == 'linear':
            selector = {'grid': self.tile.selector}
        else:
            selector = {self.config.xname: self.tile.xselector,
                        self.config.yname: self.tile.yselector}
        try:
            with xr.open_mfdataset(input_file_paths) as ds:
                if self.tile.type == 'linear':
                    # Reconfigure the grid for linear tiles.
                    ds = ds.stack(grid=(self.config.yname, self.config.xname))
                tile_ds = ds.isel(**selector)
                if in_memory:
                    tile_ds.load()
        except RuntimeError:
            msg = 'Failed to open input files "{}".'
            raise TileInitializationError(msg.format(input_file_paths))
        return tile_ds

    def run(self):
        """Start the tile's runs in serial."""
        log_file_name = 'run.{:03d}.{}.log'.format(
            self.tile.id, self.config.start_time.strftime('%Y%m%d%H%M%S'))
        log_file_path = pjoin(self.config.work_directory, log_file_name)
        # Create the work directory if it doesn't exist.
        try:
            os.makedirs(self.config.work_directory, exist_ok=True)
        except PermissionError as e:
            msg = 'Cannot create work directory "{}", permission denied.'
            raise TileRunError(msg.format(run_directory))
        with open(log_file_path, 'w') as lf:
            # Write a header to the run log file.
            header = 'Tile: {!s}`n'.format(self.tile)
            lf.write(header)
            log = get_logger(lf)
            tile_result = TileResult(id=self.tile.id, cell_results=[])
            for cell in self.tile.cells():
                cell_result = self.run_cell(cell, logger=log)
                tile_result.cell_results.append(cell_result)
            log('Finished tile #{:03d}'.format(self.tile.id))
        return tile_result

    @abstractmethod
    def run_cell(self, cell, logger=print):
        """
        Run an individual cell of the tile.

        **Argument:**

        * cell
            A `Cell` instance des cribing the cell to be run.

        **Keyword argument:**

        * logger
            A callable logging function. Defaults to `print`.

        **Returns:**

        * cell_result
            A `CellResult` object describing the result of the run.

        .. note:: This method must be implemented by the derived class.

        """
        raise NotImplementedError('run_cell() must be defined.')

    def get_cell(self, cell):
        """
        Retrieve the `xarray.Dataset` corresponding to a given cell
        belonging to the tile.

        **Arguments:**

        * cell: `scmtiles.grid_manager.Cell`
            The cell to extract from the tile.

        **Returns:**

        * cell_ds: `xarray.Dataset`
            The dataset corresponding to the required cell.

        """
        if self.tile.type == 'linear':
            selector = {'grid': cell.x}
        else:
            selector = {self.config.xname: cell.x, self.config.yname: cell.y}
        cell_ds = self._tile_data.isel(**selector)
        if self.tile.type == 'linear':
            # Convert the grid coordinate back to latitude and longitude.
            # NB: cell_ds.unstack('grid') does not appear to work.
            lat, lon = cell_ds['grid'].data
            cell_ds.coords.update({self.config.yname: lat,
                                   self.config.xname: lon})
            cell_ds = cell_ds.drop('grid')
        return cell_ds

    def create_run_directory(self):
        """
        Create a temporary run directory.

        **Returns:**

        * run_directory
            The path to the created directory.

        """
        try:
            run_directory = mkdtemp(dir=self.config.work_directory,
                                    prefix='run.')
        except PermissionError:
            msg = 'Cannot create run directory "{}", permission denied.'
            raise TileRunError(msg.format(run_directory))
        return run_directory

    def link_template(self, run_directory):
        """
        Link all filesin the template directory to the run directory.

        **Argument:**

        * run_directory
            Path to the run directory in which links will be created.

        **Returns:**

        * linked_files
            A list of 2-tuples containing the full paths to the source
            and target of each linked file.

        """
        linked_files = []
        template_pattern = '{}/*'.format(self.config.template_directory)
        for source in glob.glob(template_pattern):
            target = pjoin(run_directory, os.path.basename(source))
            try:
                os.symlink(source, target)
            except PermissionError:
                msg = 'Cannot create links in "{}", permission denied.'
                raise TileRunError(msg.format(run_directory))
            else:
                linked_files.append((source, target))
        return linked_files
<|endoftext|>