<commit_before>import imp
import os
import re
import time

from fabric.api import task, puts, env, local, sudo, run, execute, lcd
from fabric.tasks import Task
from fabric.utils import abort

try:
    import boto.ec2
    AWS_SUPPORT = True
    #import boto.ec2.elb
    #self.boto_ec2 = boto.ec2
    #self.boto_ec2_elb = boto.ec2.elb
except ImportError:
    AWS_SUPPORT = False

ip_re = re.compile('(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})')

env.aptitude = "sudo aptitude -q -y"

def _get_ec2_connection():
    return boto.ec2.connect_to_region(
        os.environ.get('AWS_REGION', 'eu-west-1'))

def _find_ec2_by_name(name, state='running'):
    ec2 = _get_ec2_connection()
    reservations = ec2.get_all_instances(
        filters={'tag:Name': name, 'instance-state-name': state})
    if len(reservations) == 0:
        return None
    elif len(reservations) == 1:
        return reservations[0].instances[0]
    puts("Found more than one!")
    return None

class BaseHost(object):

    def __init__(self, settings):
        self.settings = settings

    def get_host_user(self):
        return getattr(self.settings, 'SSH_USER', env.user)

class EC2Host(BaseHost):

    def __init__(self, *args, **kwargs):
        super(EC2Host, self).__init__(*args, **kwargs)

    def get_ip(self):
        ec2 = _find_ec2_by_name(self.settings.EC2_NAME)
        return ec2.ip_address

class VagrantHost(BaseHost):

    def __init__(self, *args, **kwargs):
        super(VagrantHost, self).__init__(*args, **kwargs)
        self.vagrant_dir = os.path.dirname(self.settings.VAGRANT_FILE)

    def get_ip(self):
        ro = re.compile(r'^\s+config.+:private_network')
        with open(self.settings.VAGRANT_FILE, 'r') as vf:
            for line in vf:
                mo = ro.search(line)
                if mo is None:
                    continue
                mo = ip_re.search(line)
                if mo is not None:
                    return mo.group('ip')
        return None

class FabTaskIt(object):
    AWS_REQUIRED_VARS = [
        'AWS_ACCESS_KEY_ID',
        'AWS_AVAILABILITY_ZONE',
        'AWS_KEYPAIR_NAME',
        'AWS_REGION',
        'AWS_SECRET_ACCESS_KEY',
    ]

    def __init__(self):
        self.fabhosts = {}
        self.fabenvdirs = {}
        self.activated_hosts = []
        self.amis = {}

    def setup(self):
        fabhome = os.path.expanduser('~/.fab-task-it')

        if os.path.exists(fabhome):
            envdirshome = os.path.join(fabhome, 'envdirs')
            if os.path.exists(envdirshome):
                self.load_envdirs(envdirshome)

            hostshome = os.path.join(fabhome, 'hosts')
            if os.path.exists(hostshome):
                self.load_hosts(hostshome)

            ec2home = os.path.join(fabhome, 'ec2')
            if os.path.exists(ec2home):
                ami_list_file = os.path.join(ec2home, 'amis.py')
                if os.path.exists(ami_list_file):
                    self.load_amis(ami_list_file)

        self.setup_hosts()
        self.setup_environments()

    def load_envdirs(self, envdir):
        for envroot, envdirs, envfiles in os.walk(envdir):
            for d in envdirs:
                fabenvdir = {
                    'name': d,
                    'path': os.path.join(envdir, d),
                }
                fabenvdir['vars'] = self._get_env_vars(fabenvdir['path'])
                self.fabenvdirs[d] = fabenvdir

    def _get_env_vars(self, path):
        env_vars = {}
        for root, dirs, files in os.walk(path):
            for filename in files:
                with open(os.path.join(path, filename)) as f:
                    env_vars[filename] = f.readline().rstrip("\n")
        return env_vars

    def load_hosts(self, hostsdir):
        for root, dirs, files in os.walk(hostsdir):
            for filename in files:
                if filename[-3:] != '.py' or filename == '__init__.py':
                    continue
                modname = filename[:-3]
                fabhost = {'name': modname}
                try:
                    fabhost['settings'] = imp.load_source(
                        'fabhosts.{}'.format(modname),
                        os.path.join(hostsdir, filename)
                    )
                    self.fabhosts[fabhost['name']] = fabhost
                    if fabhost['settings'].HOST_TYPE == 'EC2':
                        fabhost['helper'] = EC2Host(fabhost['settings'])
                    elif fabhost['settings'].HOST_TYPE == 'VAGRANT':
                        fabhost['helper'] = VagrantHost(fabhost['settings'])
                except ImportError:
                    puts("host [{}] initialization failed.".format(filename))

    def load_amis(self, filename):
        try:
            self.amis_module = imp.load_source(
                'fabamis.amis',
                filename,
            )
            self.amis = self.amis_module.AMIS
        except ImportError:
            abort("amis file [{}] initialization failed.".format(filename))

    def load_environment(self, fabenv):
        envdir = self.fabenvdirs.get(fabenv, None)
        if envdir is None:
            puts('WARNING: {} environment not found!'.format(fabenv))
            return
        for k, v in envdir['vars'].items():
            os.environ[k] = v

    def load_host_environments(self, fabhost):
        for host_env in fabhost['settings'].ENVIRONMENTS:
            self.load_environment(host_env)
        env.user = self.get_host_user(fabhost['name'])

    def get_host_port(self, hostname=None):
        if hostname is None:
            hostname = self.activated_hosts[0]
        return getattr(self.fabhosts[hostname]['settings'], 'SSH_PORT', 22)

    def get_host_user(self, hostname=None):
        if hostname is None:
            hostname = self.activated_hosts[0]
        return self.fabhosts[hostname]['helper'].get_host_user()

    def activate_host(self, host):
        fabhost = self.fabhosts[host]
        self.load_host_environments(fabhost)
        env.hosts.append(fabhost['helper'].get_ip())
        self.activated_hosts.append(fabhost['name'])

    def get_activate_host_func(self, host):
        def f():
            return self.activate_host(host)
        return f

    def setup_hosts(self):
        for i, host in enumerate(self.fabhosts.keys()):
            globals()['host_{}'.format(i)] = SimpleTask(
                self.get_activate_host_func(host),
                name='host_{}'.format(host),
            )

    def activate_environment(self, fabenv):
        self.load_environment(fabenv)

    def get_activate_env_function(self, env):
        def f():
            return self.activate_environment(env)
        return f

    def setup_environments(self):
        for i, env in enumerate(self.fabenvdirs.keys()):
            globals()['env_{}'.format(i)] = SimpleTask(
                self.get_activate_env_function(env),
                name='env_{}'.format(self.fabenvdirs[env]['name']),
            )

    def get_available_files(self, *filenames):
        result = []
        for filename in filenames:
            pth = os.path.expanduser(filename)
            if os.path.exists(pth):
                result.append(pth)
        return result

    def get_ssh_private_key_files(self):
        return self.get_available_files('~/.ssh/id_dsa', '~/.ssh/id_rsa')

    def get_ssh_public_key_files(self):
        return self.get_available_files('~/.ssh/id_dsa.pub',
                                        '~/.ssh/id_rsa.pub')

    def get_ssh_command(self):
        port = self.get_host_port()
        user = self.get_host_user()
        host = env.hosts[0]
        hostname = self.activated_hosts[0]
        ssh_key_str = getattr(self.fabhosts[hostname]['settings'],
            'SSH_KEY', ' ')
        if ssh_key_str != ' ':
            ssh_key_str = ' -i {} '.format(ssh_key_str)
        return 'ssh -p {port} -l {user}{ssh_key}{host}'.format(
            host=host,
            port=port,
            user=user,
            ssh_key=ssh_key_str,
        )

    def get_active_host_helper(self, hostname=None):
        if hostname is None:
            hostname = self.activated_hosts[0]
        return self.fabhosts[hostname]['helper']

class SimpleTask(Task):
    def __init__(self, func, *args, **kwargs):
        super(SimpleTask, self).__init__(*args, **kwargs)
        self.func = func

    def run(self, *args, **kwargs):
        return self.func(*args, **kwargs)

class AWSTask(Task):
    def __init__(self, func, *args, **kwargs):
        super(AWSTask, self).__init__(*args, **kwargs)
        self.func = func

    def run(self, *args, **kwargs):
        if not AWS_SUPPORT:
            abort('No aws support! Install boto.')
        for envvar in fabtaskit.AWS_REQUIRED_VARS:
            if envvar not in os.environ:
                abort('AWS environment not configured properly!')
        return self.func(*args, **kwargs)

fabtaskit = FabTaskIt()
fabtaskit.setup()

def print_ec2(ec2):
    title = "Name: {}".format(ec2.tags['Name'])
    puts(title)
    puts("=" * len(title))
    puts("ID: {}".format(ec2.id))
    puts("Name: {}".format(ec2.tags['Name']))
    puts("State: {}".format(ec2.state))
    puts("Type: {}".format(ec2.instance_type))
    puts("Public DNS: {}".format(ec2.dns_name))
    puts("Public IP: {}".format(ec2.ip_address))
    puts("Private DNS: {}".format(ec2.private_dns_name))
    puts("Private IP: {}".format(ec2.private_ip_address))
    # http://docs.pythonboto.org/en/latest/ref/ec2.html \
    #   #boto.ec2.instance.Instance.get_attribute
    ia = ec2.get_attribute('disableApiTermination')
    puts("Termination Protection: {}".format(ia['disableApiTermination']))
    ia = ec2.get_attribute('instanceInitiatedShutdownBehavior')
    puts("Shutdown Behavior: {}".format(
        ia['instanceInitiatedShutdownBehavior']))
    puts("")

@task
def test_hostname():
    puts(fabtaskit.activated_hosts)

@task
def test_environment():
    puts(os.environ)

@task(task_class=AWSTask)
def ec2_find_by_name(name):
    ec2 = _find_ec2_by_name(name)
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    print_ec2(ec2)

@task
def login():
    local(fabtaskit.get_ssh_command())

@task
def ec2_list_all():
    ec2 = _get_ec2_connection()
    instances = ec2.get_all_instances()
    for reservation in instances:
        print_ec2(reservation.instances[0])

@task
def supervisor_restart(service):
    sudo('supervisorctl restart {0}'.format(service))

@task
def supervisor_status():
    sudo('supervisorctl status')

@task
def tailf(filepath):
    env.output_prefix = False
    sudo('tail -f {0}'.format(filepath))
    env.output_prefix = True

@task
def copy_ssh_pub_keys():
    keys = []

    for filename in fabtaskit.get_ssh_public_key_files():
        try:
            with open(filename, 'r') as f:
                keys.append(f.readline().rstrip("\n"))
        except IOError:
            pass

    puts('Got {} keys'.format(len(keys)))

    ssh_cmd = fabtaskit.get_ssh_command()

    result = local(
        "{ssh_cmd} 'cat .ssh/authorized_keys'".format(ssh_cmd=ssh_cmd),
        capture=True,
    )
    for key in keys:
        if result.find(key) < 0:
            puts('not found key:')
            puts(key)
            local("""{ssh_cmd} 'echo "{key}" >> .ssh/authorized_keys'"""
                  .format(ssh_cmd=ssh_cmd, key=key))

@task
def aptitude_update():
    run("{0} update".format(env.aptitude), quiet=True)

@task
def aptitude_safe_upgrade():
    execute(aptitude_update)
    run("{0} safe-upgrade".format(env.aptitude))

@task
def vagrant_up():
    vagrant = fabtaskit.get_active_host_helper()
    with lcd(vagrant.vagrant_dir):
        local('vagrant up')

@task
def vagrant_halt():
    vagrant = fabtaskit.get_active_host_helper()
    with lcd(vagrant.vagrant_dir):
        local('vagrant halt')

@task(task_class=AWSTask)
def ec2_terminate(name):
    ec2 = _find_ec2_by_name(name)
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    try:
        ec2.terminate()
        puts("Instance terminated!!")
    except boto.exception.EC2ResponseError as ex:
        puts("[{}] {}".format(ex.error_code, ex.message))

@task(task_class=AWSTask)
def ec2_run_instances(ami_name, tag_name):
    ec2 = _get_ec2_connection()
    ami = fabtaskit.amis.get(ami_name)
    if ami is None:
        abort("Couldn't find ami conf with name: {}".format(ami_name))
    kwargs = ami.copy()
    kwargs['key_name'] = os.environ['AWS_KEYPAIR_NAME']
    reservation = ec2.run_instances(**kwargs)
    instance = reservation.instances[0]
    ec2.create_tags([instance.id], {'Name': tag_name})
    tries = 5
    booted = False
    while not booted and tries != 0:
        time.sleep(5)
        for r in ec2.get_all_instances():
            if r.id == reservation.id:
                print_ec2(r.instances[0])
                booted = True
                break
        tries -= 1
    if not booted:
        abort("Couldn't find if instance booted")

@task(task_class=AWSTask)
def ec2_stop_instance(name):
    ec2 = _find_ec2_by_name(name)
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    try:
        ec2.stop()
        puts("Instance stopped!!")
    except boto.exception.EC2ResponseError as ex:
        puts("[{}] {}".format(ex.error_code, ex.message))

@task(task_class=AWSTask)
def ec2_start_instance(name):
    ec2 = _find_ec2_by_name(name, state='stopped')
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    try:
        ec2.start()
        puts("Instance started!!")
    except boto.exception.EC2ResponseError as ex:
        puts("[{}] {}".format(ex.error_code, ex.message))
<commit_msg>pgdump command<commit_after>import datetime
import imp
import os
import re
import time

from fabric.api import task, puts, env, local, sudo, run, execute, lcd, get
from fabric.tasks import Task
from fabric.utils import abort

try:
    import boto.ec2
    AWS_SUPPORT = True
    #import boto.ec2.elb
    #self.boto_ec2 = boto.ec2
    #self.boto_ec2_elb = boto.ec2.elb
except ImportError:
    AWS_SUPPORT = False

ip_re = re.compile('(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})')

env.aptitude = "sudo aptitude -q -y"

def _get_ec2_connection():
    return boto.ec2.connect_to_region(
        os.environ.get('AWS_REGION', 'eu-west-1'))

def _find_ec2_by_name(name, state='running'):
    ec2 = _get_ec2_connection()
    reservations = ec2.get_all_instances(
        filters={'tag:Name': name, 'instance-state-name': state})
    if len(reservations) == 0:
        return None
    elif len(reservations) == 1:
        return reservations[0].instances[0]
    puts("Found more than one!")
    return None

class BaseHost(object):

    def __init__(self, settings):
        self.settings = settings

    def get_host_user(self):
        return getattr(self.settings, 'SSH_USER', env.user)

class EC2Host(BaseHost):

    def __init__(self, *args, **kwargs):
        super(EC2Host, self).__init__(*args, **kwargs)

    def get_ip(self):
        ec2 = _find_ec2_by_name(self.settings.EC2_NAME)
        return ec2.ip_address

class VagrantHost(BaseHost):

    def __init__(self, *args, **kwargs):
        super(VagrantHost, self).__init__(*args, **kwargs)
        self.vagrant_dir = os.path.dirname(self.settings.VAGRANT_FILE)

    def get_ip(self):
        ro = re.compile(r'^\s+config.+:private_network')
        with open(self.settings.VAGRANT_FILE, 'r') as vf:
            for line in vf:
                mo = ro.search(line)
                if mo is None:
                    continue
                mo = ip_re.search(line)
                if mo is not None:
                    return mo.group('ip')
        return None

class FabTaskIt(object):
    AWS_REQUIRED_VARS = [
        'AWS_ACCESS_KEY_ID',
        'AWS_AVAILABILITY_ZONE',
        'AWS_KEYPAIR_NAME',
        'AWS_REGION',
        'AWS_SECRET_ACCESS_KEY',
    ]

    def __init__(self):
        self.fabhosts = {}
        self.fabenvdirs = {}
        self.activated_hosts = []
        self.amis = {}

    def setup(self):
        fabhome = os.path.expanduser('~/.fab-task-it')

        if os.path.exists(fabhome):
            envdirshome = os.path.join(fabhome, 'envdirs')
            if os.path.exists(envdirshome):
                self.load_envdirs(envdirshome)

            hostshome = os.path.join(fabhome, 'hosts')
            if os.path.exists(hostshome):
                self.load_hosts(hostshome)

            ec2home = os.path.join(fabhome, 'ec2')
            if os.path.exists(ec2home):
                ami_list_file = os.path.join(ec2home, 'amis.py')
                if os.path.exists(ami_list_file):
                    self.load_amis(ami_list_file)

        self.setup_hosts()
        self.setup_environments()

    def load_envdirs(self, envdir):
        for envroot, envdirs, envfiles in os.walk(envdir):
            for d in envdirs:
                fabenvdir = {
                    'name': d,
                    'path': os.path.join(envdir, d),
                }
                fabenvdir['vars'] = self._get_env_vars(fabenvdir['path'])
                self.fabenvdirs[d] = fabenvdir

    def _get_env_vars(self, path):
        env_vars = {}
        for root, dirs, files in os.walk(path):
            for filename in files:
                with open(os.path.join(path, filename)) as f:
                    env_vars[filename] = f.readline().rstrip("\n")
        return env_vars

    def load_hosts(self, hostsdir):
        for root, dirs, files in os.walk(hostsdir):
            for filename in files:
                if filename[-3:] != '.py' or filename == '__init__.py':
                    continue
                modname = filename[:-3]
                fabhost = {'name': modname}
                try:
                    fabhost['settings'] = imp.load_source(
                        'fabhosts.{}'.format(modname),
                        os.path.join(hostsdir, filename)
                    )
                    self.fabhosts[fabhost['name']] = fabhost
                    if fabhost['settings'].HOST_TYPE == 'EC2':
                        fabhost['helper'] = EC2Host(fabhost['settings'])
                    elif fabhost['settings'].HOST_TYPE == 'VAGRANT':
                        fabhost['helper'] = VagrantHost(fabhost['settings'])
                except ImportError:
                    puts("host [{}] initialization failed.".format(filename))

    def load_amis(self, filename):
        try:
            self.amis_module = imp.load_source(
                'fabamis.amis',
                filename,
            )
            self.amis = self.amis_module.AMIS
        except ImportError:
            abort("amis file [{}] initialization failed.".format(filename))

    def load_environment(self, fabenv):
        envdir = self.fabenvdirs.get(fabenv, None)
        if envdir is None:
            puts('WARNING: {} environment not found!'.format(fabenv))
            return
        for k, v in envdir['vars'].items():
            os.environ[k] = v

    def load_host_environments(self, fabhost):
        for host_env in fabhost['settings'].ENVIRONMENTS:
            self.load_environment(host_env)
        env.user = self.get_host_user(fabhost['name'])

    def get_host_port(self, hostname=None):
        if hostname is None:
            hostname = self.activated_hosts[0]
        return getattr(self.fabhosts[hostname]['settings'], 'SSH_PORT', 22)

    def get_host_user(self, hostname=None):
        if hostname is None:
            hostname = self.activated_hosts[0]
        return self.fabhosts[hostname]['helper'].get_host_user()

    def activate_host(self, host):
        fabhost = self.fabhosts[host]
        self.load_host_environments(fabhost)
        env.hosts.append(
            '{ip}:{port}'.format(
                ip=fabhost['helper'].get_ip(),
                port=self.get_host_port(hostname=host),
            )
        )
        self.activated_hosts.append(fabhost['name'])

    def get_activate_host_func(self, host):
        def f():
            return self.activate_host(host)
        return f

    def setup_hosts(self):
        for i, host in enumerate(self.fabhosts.keys()):
            globals()['host_{}'.format(i)] = SimpleTask(
                self.get_activate_host_func(host),
                name='host_{}'.format(host),
            )

    def activate_environment(self, fabenv):
        self.load_environment(fabenv)

    def get_activate_env_function(self, env):
        def f():
            return self.activate_environment(env)
        return f

    def setup_environments(self):
        for i, env in enumerate(self.fabenvdirs.keys()):
            globals()['env_{}'.format(i)] = SimpleTask(
                self.get_activate_env_function(env),
                name='env_{}'.format(self.fabenvdirs[env]['name']),
            )

    def get_available_files(self, *filenames):
        result = []
        for filename in filenames:
            pth = os.path.expanduser(filename)
            if os.path.exists(pth):
                result.append(pth)
        return result

    def get_ssh_private_key_files(self):
        return self.get_available_files('~/.ssh/id_dsa', '~/.ssh/id_rsa')

    def get_ssh_public_key_files(self):
        return self.get_available_files('~/.ssh/id_dsa.pub',
                                        '~/.ssh/id_rsa.pub')

    def get_ssh_command(self):
        user = self.get_host_user()
        if ':' in env.hosts[0]:
            (host, port) = env.hosts[0].split(':')
        else:
            host = env.hosts[0]
            port = self.get_host_port()
        hostname = self.activated_hosts[0]
        ssh_key_str = getattr(self.fabhosts[hostname]['settings'],
            'SSH_KEY', ' ')
        if ssh_key_str != ' ':
            ssh_key_str = ' -i {} '.format(ssh_key_str)
        return 'ssh -p {port} -l {user}{ssh_key}{host}'.format(
            host=host,
            port=port,
            user=user,
            ssh_key=ssh_key_str,
        )

    def get_active_host_helper(self, hostname=None):
        if hostname is None:
            hostname = self.activated_hosts[0]
        return self.fabhosts[hostname]['helper']

class SimpleTask(Task):
    def __init__(self, func, *args, **kwargs):
        super(SimpleTask, self).__init__(*args, **kwargs)
        self.func = func

    def run(self, *args, **kwargs):
        return self.func(*args, **kwargs)

class AWSTask(Task):
    def __init__(self, func, *args, **kwargs):
        super(AWSTask, self).__init__(*args, **kwargs)
        self.func = func

    def run(self, *args, **kwargs):
        if not AWS_SUPPORT:
            abort('No aws support! Install boto.')
        for envvar in fabtaskit.AWS_REQUIRED_VARS:
            if envvar not in os.environ:
                abort('AWS environment not configured properly!')
        return self.func(*args, **kwargs)

fabtaskit = FabTaskIt()
fabtaskit.setup()

def print_ec2(ec2):
    title = "Name: {}".format(ec2.tags['Name'])
    puts(title)
    puts("=" * len(title))
    puts("ID: {}".format(ec2.id))
    puts("Name: {}".format(ec2.tags['Name']))
    puts("State: {}".format(ec2.state))
    puts("Type: {}".format(ec2.instance_type))
    puts("Public DNS: {}".format(ec2.dns_name))
    puts("Public IP: {}".format(ec2.ip_address))
    puts("Private DNS: {}".format(ec2.private_dns_name))
    puts("Private IP: {}".format(ec2.private_ip_address))
    # http://docs.pythonboto.org/en/latest/ref/ec2.html \
    #   #boto.ec2.instance.Instance.get_attribute
    ia = ec2.get_attribute('disableApiTermination')
    puts("Termination Protection: {}".format(ia['disableApiTermination']))
    ia = ec2.get_attribute('instanceInitiatedShutdownBehavior')
    puts("Shutdown Behavior: {}".format(
        ia['instanceInitiatedShutdownBehavior']))
    puts("")

@task
def test_hostname():
    puts(fabtaskit.activated_hosts)

@task
def test_environment():
    puts(os.environ)

@task(task_class=AWSTask)
def ec2_find_by_name(name):
    ec2 = _find_ec2_by_name(name)
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    print_ec2(ec2)

@task
def login():
    local(fabtaskit.get_ssh_command())

@task
def ec2_list_all():
    ec2 = _get_ec2_connection()
    instances = ec2.get_all_instances()
    for reservation in instances:
        print_ec2(reservation.instances[0])

@task
def supervisor_restart(service):
    sudo('supervisorctl restart {0}'.format(service))

@task
def supervisor_status():
    sudo('supervisorctl status')

@task
def tailf(filepath):
    env.output_prefix = False
    sudo('tail -f {0}'.format(filepath))
    env.output_prefix = True

@task
def copy_ssh_pub_keys():
    keys = []

    for filename in fabtaskit.get_ssh_public_key_files():
        try:
            with open(filename, 'r') as f:
                keys.append(f.readline().rstrip("\n"))
        except IOError:
            pass

    puts('Got {} keys'.format(len(keys)))

    ssh_cmd = fabtaskit.get_ssh_command()

    result = local(
        "{ssh_cmd} 'cat .ssh/authorized_keys'".format(ssh_cmd=ssh_cmd),
        capture=True,
    )
    for key in keys:
        if result.find(key) < 0:
            puts('not found key:')
            puts(key)
            local("""{ssh_cmd} 'echo "{key}" >> .ssh/authorized_keys'"""
                  .format(ssh_cmd=ssh_cmd, key=key))

@task
def aptitude_update():
    run("{0} update".format(env.aptitude), quiet=True)

@task
def aptitude_safe_upgrade():
    execute(aptitude_update)
    run("{0} safe-upgrade".format(env.aptitude))

@task
def vagrant_up():
    vagrant = fabtaskit.get_active_host_helper()
    with lcd(vagrant.vagrant_dir):
        local('vagrant up')

@task
def vagrant_halt():
    vagrant = fabtaskit.get_active_host_helper()
    with lcd(vagrant.vagrant_dir):
        local('vagrant halt')

@task(task_class=AWSTask)
def ec2_terminate(name):
    ec2 = _find_ec2_by_name(name)
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    try:
        ec2.terminate()
        puts("Instance terminated!!")
    except boto.exception.EC2ResponseError as ex:
        puts("[{}] {}".format(ex.error_code, ex.message))

@task(task_class=AWSTask)
def ec2_run_instances(ami_name, tag_name):
    ec2 = _get_ec2_connection()
    ami = fabtaskit.amis.get(ami_name)
    if ami is None:
        abort("Couldn't find ami conf with name: {}".format(ami_name))
    kwargs = ami.copy()
    kwargs['key_name'] = os.environ['AWS_KEYPAIR_NAME']
    reservation = ec2.run_instances(**kwargs)
    instance = reservation.instances[0]
    ec2.create_tags([instance.id], {'Name': tag_name})
    tries = 5
    booted = False
    while not booted and tries != 0:
        time.sleep(5)
        for r in ec2.get_all_instances():
            if r.id == reservation.id:
                print_ec2(r.instances[0])
                booted = True
                break
        tries -= 1
    if not booted:
        abort("Couldn't find if instance booted")

@task(task_class=AWSTask)
def ec2_stop_instance(name):
    ec2 = _find_ec2_by_name(name)
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    try:
        ec2.stop()
        puts("Instance stopped!!")
    except boto.exception.EC2ResponseError as ex:
        puts("[{}] {}".format(ex.error_code, ex.message))

@task(task_class=AWSTask)
def ec2_start_instance(name):
    ec2 = _find_ec2_by_name(name, state='stopped')
    if ec2 is None:
        abort("Couldn't find '{}'".format(name))
    try:
        ec2.start()
        puts("Instance started!!")
    except boto.exception.EC2ResponseError as ex:
        puts("[{}] {}".format(ex.error_code, ex.message))

@task
def pg_dump_gz(db, user='postgres'):
    today = datetime.date.today()
    filename = '{year}{month:02}{day:02}_{dbname}.sql.gz'.format(
        year=today.year, month=today.month, day=today.day, dbname=db)
    #env.output_prefix = False
    sudo(
        'pg_dump {db} | gzip > /tmp/{filename}'.format(db=db,
                                                       filename=filename),
        user=user)
    get('/tmp/{}'.format(filename), './{}'.format(filename))
    #env.output_prefix = True
<|endoftext|>


<commit_before>import os
import ast

from fabfile.config import *

def copy_pkg(tgt_host, pkg_file):
    with settings(host_string = tgt_host):
        put(pkg_file, use_sudo=True)
#end _copy_pkg

def install_pkg(tgt_host, pkg_file):
    with settings(host_string = tgt_host):
        sudo("rpm -iv --force %s" %(pkg_file.split('/')[-1]))
#end _install_pkg

def get_linux_distro():
    linux_distro = "python -c 'from platform import linux_distribution; print linux_distribution()'"
    (dist, version, extra) = ast.literal_eval(sudo(linux_distro))
    return (dist, version, extra)

def detect_ostype():
    (dist, version, extra) = get_linux_distro()
    if extra is not None and 'xen' in extra:
        dist = 'xen'
    if 'red hat' in dist.lower():
        dist = 'redhat'
    return dist.lower()
#end detect_ostype

def get_release(pkg='contrail-install-packages'):
    pkg_ver = None
    dist = detect_ostype()
    if dist in ['centos', 'fedora', 'redhat']:
        cmd = "rpm -q --queryformat '%%{VERSION}' %s" %pkg
    elif dist in ['ubuntu']:
        cmd = "dpkg -s %s | grep Version: | cut -d' ' -f2 | cut -d'-' -f1" %pkg
    pkg_ver = sudo(cmd)
    if 'is not installed' in pkg_ver or 'is not available' in pkg_ver:
        print "Package %s not installed." % pkg
        return None
    return pkg_ver

def get_build(pkg='contrail-install-packages'):
    pkg_rel = None
    dist = detect_ostype()
    if dist in ['centos', 'fedora', 'redhat']:
        cmd = "rpm -q --queryformat '%%{RELEASE}' %s" %pkg
    elif dist in ['ubuntu']:
        cmd = "dpkg -s %s | grep Version: | cut -d' ' -f2 | cut -d'-' -f2" %pkg
    pkg_rel = sudo(cmd)
    if 'is not installed' in pkg_rel or 'is not available' in pkg_rel:
        print "Package %s not installed." % pkg
        return None
    return pkg_rel

def is_package_installed(pkg_name):
    ostype = detect_ostype()
    if ostype in ['ubuntu']:
        cmd = 'dpkg-query -l "%s" | grep -q ^.i'
    elif ostype in ['centos','fedora']:
        cmd = 'rpm -qi %s '
    cmd = cmd % (pkg_name)
    with settings(warn_only=True):
        result = sudo(cmd)
    return result.succeeded
#end is_package_installed

def get_as_sudo(src_file, dst_file):
    basename = os.path.basename(src_file.rstrip('/'))
    if not basename or basename in ['*']:
        raise Exception('%s is not a valid path'%src_file)
    tmp_file = '/tmp/tmp%s'%basename
    sudo('cp -rf %s %s'%(src_file, tmp_file))
    sudo('chmod -R 777 %s'%tmp_file)
    try:
        status = get(tmp_file, dst_file)
    except:
        sudo('rm -rf %s'%tmp_file)
        raise
    sudo('rm -rf %s'%tmp_file)
<commit_msg>rename tmp<file> to <file> if destination is a directory<commit_after>import os
import ast

from fabfile.config import *

def copy_pkg(tgt_host, pkg_file):
    with settings(host_string = tgt_host):
        put(pkg_file, use_sudo=True)
#end _copy_pkg

def install_pkg(tgt_host, pkg_file):
    with settings(host_string = tgt_host):
        sudo("rpm -iv --force %s" %(pkg_file.split('/')[-1]))
#end _install_pkg

def get_linux_distro():
    linux_distro = "python -c 'from platform import linux_distribution; print linux_distribution()'"
    (dist, version, extra) = ast.literal_eval(sudo(linux_distro))
    return (dist, version, extra)

def detect_ostype():
    (dist, version, extra) = get_linux_distro()
    if extra is not None and 'xen' in extra:
        dist = 'xen'
    if 'red hat' in dist.lower():
        dist = 'redhat'
    return dist.lower()
#end detect_ostype

def get_release(pkg='contrail-install-packages'):
    pkg_ver = None
    dist = detect_ostype()
    if dist in ['centos', 'fedora', 'redhat']:
        cmd = "rpm -q --queryformat '%%{VERSION}' %s" %pkg
    elif dist in ['ubuntu']:
        cmd = "dpkg -s %s | grep Version: | cut -d' ' -f2 | cut -d'-' -f1" %pkg
    pkg_ver = sudo(cmd)
    if 'is not installed' in pkg_ver or 'is not available' in pkg_ver:
        print "Package %s not installed." % pkg
        return None
    return pkg_ver

def get_build(pkg='contrail-install-packages'):
    pkg_rel = None
    dist = detect_ostype()
    if dist in ['centos', 'fedora', 'redhat']:
        cmd = "rpm -q --queryformat '%%{RELEASE}' %s" %pkg
    elif dist in ['ubuntu']:
        cmd = "dpkg -s %s | grep Version: | cut -d' ' -f2 | cut -d'-' -f2" %pkg
    pkg_rel = sudo(cmd)
    if 'is not installed' in pkg_rel or 'is not available' in pkg_rel:
        print "Package %s not installed." % pkg
        return None
    return pkg_rel

def is_package_installed(pkg_name):
    ostype = detect_ostype()
    if ostype in ['ubuntu']:
        cmd = 'dpkg-query -l "%s" | grep -q ^.i'
    elif ostype in ['centos','fedora']:
        cmd = 'rpm -qi %s '
    cmd = cmd % (pkg_name)
    with settings(warn_only=True):
        result = sudo(cmd)
    return result.succeeded
#end is_package_installed

def get_as_sudo(src_file, dst_file):
    basename = os.path.basename(src_file.rstrip('/'))
    if not basename or basename in ['*']:
        raise Exception('%s is not a valid path'%src_file)
    tmp_file = '/tmp/tmp%s'%basename
    sudo('cp -rf %s %s'%(src_file, tmp_file))
    sudo('chmod -R 777 %s'%tmp_file)
    try:
        status = get(tmp_file, dst_file)
        if os.path.isdir(dst_file):
            local('mv %s/tmp%s %s/%s' %(dst_file, basename, dst_file, basename))
    except:
        sudo('rm -rf %s'%tmp_file)
        raise
    sudo('rm -rf %s'%tmp_file)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
#
# Author: Massimo Menichinelli
# Homepage: http://www.openp2pdesign.org
# License: MIT
#

import xlsxwriter

# Create document -------------------------------------------------------------

# Create the file
workbook = xlsxwriter.Workbook('FabLab-BusinessPlan.xlsx')

# Create the worksheets
expenses = workbook.add_worksheet('Expenses')
activities = workbook.add_worksheet('Activities')
membership = workbook.add_worksheet('Membership')
total = workbook.add_worksheet('Total')

# Create styles -------------------------------------------------------------

# Add a bold style to highlight heading cells
bold_style = workbook.add_format()
bold_style.set_font_color('white')
bold_style.set_bg_color('F56A2F')
bold_style.set_bold()

# Add a total style to highlight total cells
total_style = workbook.add_format()
total_style.set_font_color('red')
total_style.set_bg_color('FAECC5')
total_style.set_bold()

# Add a style for money
money_style = workbook.add_format()
money_style.set_num_format('$ 0.00')

# Add a style for money totals
money_total_style = workbook.add_format()
# Add green/red color for positive/negative numbers
money_total_style.set_num_format('[Green]$ 0.00;[Red]$ -0.00;$ 0.00')

# Add content -------------------------------------------------------------

# Dummy values
item_value = 0.00

# Set the width of the columns
expenses.set_column('A:A', 10)

# Add content to the Expenses worksheet
expenses.write('A1', 'Hello world', bold_style)
expenses.write('A2', item_value, money_style)
expenses.write('A3', item_value, money_style)
expenses.write('A4', '=SUM(A2:A3)', money_total_style)

# Save document -------------------------------------------------------------

# Save and close the file
workbook.close()<commit_msg>Add a debug part that open the resulting xlsx file on OSX<commit_after># -*- encoding: utf-8 -*-
#
# Author: Massimo Menichinelli
# Homepage: http://www.openp2pdesign.org
# License: MIT
#

import xlsxwriter

# Create document -------------------------------------------------------------

# Create the file
workbook = xlsxwriter.Workbook('FabLab-BusinessPlan.xlsx')

# Create the worksheets
expenses = workbook.add_worksheet('Expenses')
activities = workbook.add_worksheet('Activities')
membership = workbook.add_worksheet('Membership')
total = workbook.add_worksheet('Total')

# Create styles -------------------------------------------------------------

# Add a bold style to highlight heading cells
bold_style = workbook.add_format()
bold_style.set_font_color('white')
bold_style.set_bg_color('F56A2F')
bold_style.set_bold()

# Add a total style to highlight total cells
total_style = workbook.add_format()
total_style.set_font_color('red')
total_style.set_bg_color('FAECC5')
total_style.set_bold()

# Add a style for money
money_style = workbook.add_format()
money_style.set_num_format('$ 0.00')

# Add a style for money totals
money_total_style = workbook.add_format()
# Add green/red color for positive/negative numbers
money_total_style.set_num_format('[Green]$ 0.00;[Red]$ -0.00;$ 0.00')
total_style.set_bg_color('FAECC5')

# Add content -------------------------------------------------------------

# Dummy values
item_value = 0.00

# Set the width of the columns
expenses.set_column('A:A', 10)

# Add content to the Expenses worksheet
expenses.write('A1', 'Hello world', bold_style)
expenses.write('A2', item_value, money_style)
expenses.write('A3', item_value, money_style)
expenses.write('A4', '=SUM(A2:A3)', money_total_style)

# Save document -------------------------------------------------------------

# Save and close the file
workbook.close()

# Open the file in OSX, for debug
import subprocess
import os
where = os.path.dirname(os.path.abspath(__file__))
subprocess.check_output(["./soffice",where+"/FabLab-BusinessPlan.xlsx"], cwd="/Applications/LibreOffice.app/Contents/MacOS")<|endoftext|>


<commit_before>import ptypes, datetime, time
from ptypes import pstruct,parray,ptype,dyn,pstr,pint,pfloat,pbinary

## primitives
byte = dyn.clone(pint.uint8_t)
word = dyn.clone(pint.uint16_t)
dword = dyn.clone(pint.uint32_t)
float = dyn.clone(pfloat.single)
double = dyn.clone(pfloat.double)

uint0 = dyn.clone(pint.uint_t)
int0 = dyn.clone(pint.int_t)
uint8 = dyn.clone(pint.uint8_t)
int8 = dyn.clone(pint.int8_t)
int16 = dyn.clone(pint.int16_t)
uint16 = dyn.clone(pint.uint16_t)
int32 = dyn.clone(pint.int32_t)
uint32 = dyn.clone(pint.uint32_t)
uint64 = dyn.clone(pint.uint64_t)

class off_t(pint.uint32_t): pass
class addr_t(pint.uint32_t): pass

### locating particular parts of the executable
def LocateBase(self):
    """Return the base object of the executable. This is used to find the base address."""
    try:
        nth = self.getparent(ptype.boundary)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

def LocateHeader(self):
    """Return the executable sub-header. This will return the executable main header."""
    try:
        nth = self.getparent(Header)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

## types of relative pointers in the executable
def CalculateRelativeAddress(self, address):
    """given a va, returns offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

def CalculateRelativeOffset(self, offset):
    """return an offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        return base + offset

    # memory
    pe = LocateHeader(self)
    section = pe['Sections'].getsectionbyoffset(offset)
    o = offset - section['PointerToRawData'].int()
    return base + section['VirtualAddress'].int() + o

def CalculateRealAddress(self, address):
    """given an rva, return offset relative to the baseaddress"""
    header = LocateHeader(self)
    base = LocateBase(header)
    address -= header['OptionalHeader']['ImageBase'].int()
    base = base.getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

## ptypes closures
def realaddress(target, **kwds):
    """Returns a pointer to /target/ where value is an rva"""
    kwds.setdefault('__name__', 'realaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRealAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRealAddress, **kwds)

def fileoffset(target, **kwds):
    """Returns a pointer to /target/ where value is a fileoffset"""
    kwds.setdefault('__name__', 'fileoffset')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeOffset, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeOffset, **kwds)
def virtualaddress(target, **kwds):
    """Returns a pointer to /target/ where value is a va"""
    kwds.setdefault('__name__', 'virtualaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeAddress, **kwds)

## core header types
class Header(object): pass

class Machine(pint.enum, pint.uint16_t):
    _values_ = [
        ('UNKNOWN', 0x0000), ('AM33', 0x01d3), ('AMD64', 0x8664), ('ARM', 0x01c0),
        ('EBC', 0x0ebc), ('I386', 0x014c), ('IA64', 0x0200), ('M32R', 0x9041),
        ('MIPS16', 0x0266), ('MIPSFPU', 0x0366), ('MIPSFPU16', 0x0466), ('POWERPC', 0x01f0),
        ('POWERPCFP', 0x01f1), ('R4000', 0x0166), ('SH3', 0x01a2), ('SH3DSP', 0x01a3),
        ('SH4', 0x01a6), ('SH5', 0x01a8), ('THUMB', 0x01c2), ('WCEMIPSV2', 0x0169),
    ]

class TimeDateStamp(uint32):
    def datetime(self):
        res = self.int()
        epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
        delta = datetime.timedelta(seconds=res)
        return epoch + delta
    def details(self):
        tzinfo = datetime.timezone(datetime.timedelta(seconds=-(time.altzone if time.daylight else time.timezone)))
        res = self.datetime().astimezone(tzinfo)
        return res.isoformat()
    def summary(self):
        tzinfo = datetime.timezone(datetime.timedelta(seconds=-(time.altzone if time.daylight else time.timezone)))
        res = self.datetime().astimezone(tzinfo)
        return '({:#0{:d}x}) {!s}'.format(self.int(), 2 + 2 * self.size(), res)

class IMAGE_COMDAT_SELECT(ptypes.pint.enum, byte):
    _values_ = [
        ('NODUPIC', 1),
        ('ANY', 2),
        ('SAME_SIZE', 3),
        ('EXACT_MATCH', 4),
        ('ASSOCIATIVE', 5),
        ('LARGEST', 6)
    ]

class rfc4122(pstruct.type):
    class _Data1(pint.bigendian(pint.uint32_t)):
        def str(self):
            res = 2 * self.size()
            return "{:0{:d}x}".format(self.int(), res)
        def summary(self):
            return self.str()

    class _Data2and3(pint.bigendian(pint.uint16_t)):
        def str(self):
            res = 2 * self.size()
            return "{:0{:d}x}".format(self.int(), res)
        def summary(self):
            return self.str()

    class _Data4(pint.bigendian(pint.uint64_t)):
        def str(self):
            res = bytearray(self.serialize())
            d1 = ''.join(map('{:02x}'.format, res[:2]))
            d2 = ''.join(map('{:02x}'.format, res[2:]))
            return '-'.join([d1, d2])
        def summary(self):
            return self.str()

    _fields_ = [
        (_Data1, 'Data1'),
        (_Data2and3, 'Data2'),
        (_Data2and3, 'Data3'),
        (_Data4, 'Data4'),
    ]

    def summary(self, **options):
        if self.initializedQ():
            return '{{Data1-Data2-Data3-Data4}} {:s}'.format(self.str())
        return '{{Data1-Data2-Data3-Data4}} {{????????-????-????-????-????????????}}'

    def str(self):
        d1 = '{:08x}'.format(self['Data1'].int())
        d2 = '{:04x}'.format(self['Data2'].int())
        d3 = '{:04x}'.format(self['Data3'].int())
        res = bytearray(self['Data4'].serialize())
        d4 = ''.join(map('{:02x}'.format,res[:2]))
        d5 = ''.join(map('{:02x}'.format,res[2:]))
        return '{{{:s}}}'.format('-'.join((d1,d2,d3,d4,d5)))

class GUID(rfc4122):
    _fields_ = [
        (bo(t), n) for bo, (t, n) in zip((pint.littleendian, pint.littleendian, pint.littleendian, pint.bigendian), rfc4122._fields_)
    ]
<commit_msg>Added some microsoft-y primitive types to the `pecoff.headers` library module.<commit_after>import ptypes, datetime, time
from ptypes import pstruct,parray,ptype,dyn,pstr,pint,pfloat,pbinary

## primitives
byte = dyn.clone(pint.uint8_t)
word = dyn.clone(pint.uint16_t)
dword = dyn.clone(pint.uint32_t)
float = dyn.clone(pfloat.single)
double = dyn.clone(pfloat.double)

uint0 = dyn.clone(pint.uint_t)
int0 = dyn.clone(pint.int_t)
uint8 = dyn.clone(pint.uint8_t)
int8 = dyn.clone(pint.int8_t)
int16 = dyn.clone(pint.int16_t)
uint16 = dyn.clone(pint.uint16_t)
int32 = dyn.clone(pint.int32_t)
uint32 = dyn.clone(pint.uint32_t)
uint64 = dyn.clone(pint.uint64_t)

class off_t(pint.uint32_t): pass
class addr_t(pint.uint32_t): pass

class VOID(ptype.undefined): pass
class BYTE(pint.uint8_t): pass
class WORD(pint.uint16_t): pass
class DWORD(pint.uint32_t): pass
class ULONGLONG(pint.uint64_t): pass

### locating particular parts of the executable
def LocateBase(self):
    """Return the base object of the executable. This is used to find the base address."""
    try:
        nth = self.getparent(ptype.boundary)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

def LocateHeader(self):
    """Return the executable sub-header. This will return the executable main header."""
    try:
        nth = self.getparent(Header)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

## types of relative pointers in the executable
def CalculateRelativeAddress(self, address):
    """given a va, returns offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

def CalculateRelativeOffset(self, offset):
    """return an offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        return base + offset

    # memory
    pe = LocateHeader(self)
    section = pe['Sections'].getsectionbyoffset(offset)
    o = offset - section['PointerToRawData'].int()
    return base + section['VirtualAddress'].int() + o

def CalculateRealAddress(self, address):
    """given an rva, return offset relative to the baseaddress"""
    header = LocateHeader(self)
    base = LocateBase(header)
    address -= header['OptionalHeader']['ImageBase'].int()
    base = base.getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

## ptypes closures
def realaddress(target, **kwds):
    """Returns a pointer to /target/ where value is an rva"""
    kwds.setdefault('__name__', 'realaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRealAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRealAddress, **kwds)

def fileoffset(target, **kwds):
    """Returns a pointer to /target/ where value is a fileoffset"""
    kwds.setdefault('__name__', 'fileoffset')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeOffset, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeOffset, **kwds)
def virtualaddress(target, **kwds):
    """Returns a pointer to /target/ where value is a va"""
    kwds.setdefault('__name__', 'virtualaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeAddress, **kwds)

## core header types
class Header(object): pass

class Machine(pint.enum, WORD):
    _values_ = [
        ('UNKNOWN', 0x0000), ('AM33', 0x01d3), ('AMD64', 0x8664), ('ARM', 0x01c0),
        ('EBC', 0x0ebc), ('I386', 0x014c), ('IA64', 0x0200), ('M32R', 0x9041),
        ('MIPS16', 0x0266), ('MIPSFPU', 0x0366), ('MIPSFPU16', 0x0466), ('POWERPC', 0x01f0),
        ('POWERPCFP', 0x01f1), ('R4000', 0x0166), ('SH3', 0x01a2), ('SH3DSP', 0x01a3),
        ('SH4', 0x01a6), ('SH5', 0x01a8), ('THUMB', 0x01c2), ('WCEMIPSV2', 0x0169),
    ]

class TimeDateStamp(DWORD):
    def datetime(self):
        res = self.int()
        epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
        delta = datetime.timedelta(seconds=res)
        return epoch + delta
    def details(self):
        tzinfo = datetime.timezone(datetime.timedelta(seconds=-(time.altzone if time.daylight else time.timezone)))
        res = self.datetime().astimezone(tzinfo)
        return res.isoformat()
    def summary(self):
        tzinfo = datetime.timezone(datetime.timedelta(seconds=-(time.altzone if time.daylight else time.timezone)))
        res = self.datetime().astimezone(tzinfo)
        return '({:#0{:d}x}) {!s}'.format(self.int(), 2 + 2 * self.size(), res)

class IMAGE_COMDAT_SELECT(ptypes.pint.enum, byte):
    _values_ = [
        ('NODUPIC', 1),
        ('ANY', 2),
        ('SAME_SIZE', 3),
        ('EXACT_MATCH', 4),
        ('ASSOCIATIVE', 5),
        ('LARGEST', 6)
    ]

class rfc4122(pstruct.type):
    class _Data1(pint.bigendian(pint.uint32_t)):
        def str(self):
            res = 2 * self.size()
            return "{:0{:d}x}".format(self.int(), res)
        def summary(self):
            return self.str()

    class _Data2and3(pint.bigendian(pint.uint16_t)):
        def str(self):
            res = 2 * self.size()
            return "{:0{:d}x}".format(self.int(), res)
        def summary(self):
            return self.str()

    class _Data4(pint.bigendian(pint.uint64_t)):
        def str(self):
            res = bytearray(self.serialize())
            d1 = ''.join(map('{:02x}'.format, res[:2]))
            d2 = ''.join(map('{:02x}'.format, res[2:]))
            return '-'.join([d1, d2])
        def summary(self):
            return self.str()

    _fields_ = [
        (_Data1, 'Data1'),
        (_Data2and3, 'Data2'),
        (_Data2and3, 'Data3'),
        (_Data4, 'Data4'),
    ]

    def summary(self, **options):
        if self.initializedQ():
            return '{{Data1-Data2-Data3-Data4}} {:s}'.format(self.str())
        return '{{Data1-Data2-Data3-Data4}} {{????????-????-????-????-????????????}}'

    def str(self):
        d1 = '{:08x}'.format(self['Data1'].int())
        d2 = '{:04x}'.format(self['Data2'].int())
        d3 = '{:04x}'.format(self['Data3'].int())
        res = bytearray(self['Data4'].serialize())
        d4 = ''.join(map('{:02x}'.format,res[:2]))
        d5 = ''.join(map('{:02x}'.format,res[2:]))
        return '{{{:s}}}'.format('-'.join((d1,d2,d3,d4,d5)))

class GUID(rfc4122):
    _fields_ = [
        (bo(t), n) for bo, (t, n) in zip((pint.littleendian, pint.littleendian, pint.littleendian, pint.bigendian), rfc4122._fields_)
    ]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
fabrik_cli
------------
:copyright: (c) 2015 by Fröjd Interactive AB
:license: MIT, see LICENSE for more details.
"""

__title__ = "fabrik_cli"
__version__ = "2.0.1"
__build__ = 201
__license__ = "MIT"
__copyright__ = "Copyright 2015 Fröjd Interactive AB"
<commit_msg>Updated licensing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
fabrik_cli
------------
:copyright: (c) 2015-2016 by Fröjd Interactive AB
:license: MIT, see LICENSE for more details.
"""

__title__ = "fabrik_cli"
__version__ = "2.0.1"
__build__ = 201
__license__ = "MIT"
__copyright__ = "Copyright 2015-2016 Fröjd Interactive AB"
<|endoftext|>


<commit_before><commit_msg>clean up directories<commit_after><|endoftext|>


<commit_before>import os
import time
import json
import logging
import socket
import yaml
import re
from collections import OrderedDict

import pymongo
import requests
import pandas as pd
from tornado import ioloop, web, httpserver, process, httpclient, escape

__version__ = '0.2.0'

class Kwargs(object):
    def __init__(self, **kwargs):
        for key, val in kwargs.items():
            self.__setattr__(key, val)

class WebMethod(object):
    def __init__(self, method, url):
        self.method = method
        self.url = url
        self.params = re.findall("{(.*?)}", self.url)
        self.__doc__ = (
            '\nParameters\n----------\n' +
            '\n'.join(["{} : str".format(x) for x in self.params]))

    def __call__(self, data='', **kwargs):
        response = requests.request(
            method=self.method,
            url=self.url.format(**kwargs),
            data=pd.json.dumps(data),
            )
        if not response.ok:
            logging.warning('Error in {} {}'.format(self.method, self.url.format(**kwargs)))
            raise web.HTTPError(response.status_code, response.reason)
        return response

class Config(object):
    def __init__(self, filename):
        self.conf = yaml.load(open(filename))
        self.mongo = Kwargs()
        _mongo = self.conf.get('db', {}).get('mongo', {})
        self.mongo = Kwargs(**{
            collname: pymongo.MongoClient(host['address'],
                                          connect=False)[db['name']][coll['name']]
            for hostname, host in _mongo.get('host', {}).items()
            for dbname, db in _mongo.get('database', {}).items() if db['host'] == hostname
            for collname, coll in _mongo.get('collection', {}).items() if coll['database'] == dbname
            })
        self.services = Kwargs(**{
            key: Kwargs(**{
                subkey: Kwargs(**{
                    subsubkey: WebMethod(
                        subsubkey,
                        self.conf['registry']['url'].rstrip('/')+subsubval,
                        )
                    for subsubkey, subsubval in subval.items()})
                for subkey, subval in val.items()
                })
            for key, val in self.conf.get('services', {}).items()})

    def get_port(self):
        if 'port' not in self.conf:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.bind(("", 0))
            self.conf['port'] = s.getsockname()[1]
            s.close()
        return self.conf['port']

    def register(self):
        request = httpclient.HTTPRequest(
            '{}/register/{}'.format(
                self.conf['registry']['url'].rstrip('/'),
                self.conf['name'],
                ),
            method='POST',
            body=json.dumps({
                'url': 'http://{}:{}'.format(socket.gethostname(),
                                             self.get_port()),
                'config': self.conf,
                }),
            )
        client = httpclient.HTTPClient()
        r = client.fetch(request, raise_error=False)
        logging.debug('HEARTBEAT : {} ({}).'.format(
                r.code, r.reason[:30]))

class Callback(object):
    def __init__(self, config, uri, sleep_duration=0):
        self.config = config
        self.uri = uri
        self.sleep_duration = sleep_duration

    def __call__(self):
        logging.debug('{} callback started'.format(self.uri))
        r = requests.post('http://localhost:{}/{}'.format(
                self.config.get_port(),
                self.uri.lstrip('/')))
        if r.status_code != 200:
            logging.debug('{} callback returned {}. Sleep for a while.'.format(
                self.uri, r.status_code))
            time.sleep(self.sleep_duration)
        logging.debug('{} callback finished : {}'.format(self.uri, r.text))

class Info(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Get the information on the service's parameters.",
                "method": "GET",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def get(self):
        self.write(self.application.config.conf)

class Heartbeat(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Tell the registry that the service is alive.",
                "method": "POST",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def post(self):
        self.application.config.register()
        self.write("ok")

class Todo(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Update the list of tasks to be done.",
                "method": "POST",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def post(self):
        out = self.todo()
        if out['nb'] == 0:
            self.set_status(201)  # Nothing to do.
        self.write(json.dumps(out))

    def todo_loop(self, data):
        raise NotImplementedError()

    def todo(self):
        nb_created_tasks = 0

        # We get the `todo` task.
        r = self.application.config.services.tasks.action.put(
            task=self.application.config.conf['tasks']['todo'],
            key=self.application.config.conf['tasks']['todo'],
            action='stack',
            data={},
            )

        logging.debug('TODO : Start scanning for new tasks')
        nb_loops = 0

        while True:
            # Get and self-assign the task.
            r = self.application.config.services.tasks.assignOne.put(
                    task=self.application.config.conf['tasks']['todo'])
            if r.status_code != 200:
                break

            try:
                task = r.json()

                # If `lastScanObjectId` is not defined, we start from 1970-01-01.
                data = task['data']
                data['nb'] = data.get('nb', 0)

                # Get all documents after `lastScanObjectId`
                # #########################################
                todo_tasks = list(self.todo_loop(data))
                logging.debug('TODO : Found {} tasks'.format(len(todo_tasks)))
                for task_key, task_data in todo_tasks:
                    logging.debug('TODO : Set task {}/{}'.format(task_key, task_data))
                    r = self.application.config.services.tasks.action.put(
                        task=self.application.config.conf['tasks']['do'],
                        key=escape.url_escape(task_key),
                        action='stack',
                        data=task_data,
                        )
                    nb_created_tasks += 1

                # Update the task to `done` if nothing happenned since last GET.
                r = self.application.config.services.tasks.action.put(
                    task=self.application.config.conf['tasks']['todo'],
                    key=self.application.config.conf['tasks']['todo'],
                    action='success',
                    data=data,
                    )
            except Exception as e:
                # Update the task to `done` if nothing happenned since last GET.
                r = self.application.config.services.tasks.action.put(
                    task=self.application.config.conf['tasks']['todo'],
                    key=self.application.config.conf['tasks']['todo'],
                    action='error',
                    data={},
                    )
                logging.exception('TODO : Failed todoing.')
                return {'nb': 0, 'ok': False, 'reason': e.__repr__()}

            nb_loops += 1

        log_str = 'TODO : Finished scanning for new tasks. Found {} in {} loops.'.format(
            nb_created_tasks, nb_loops)
        if nb_created_tasks > 0:
            logging.info(log_str)
        else:
            logging.debug(log_str)

        return {'nb': nb_created_tasks, 'nbLoops': nb_loops}

class Do(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Pick one task and do it.",
                "method": "POST",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def post(self):
        out = self.do()
        if out['nb'] == 0:
            self.set_status(201)  # Nothing to do.
        self.write(json.dumps(out))

    def do_something(self, task_key, task_data):
        raise NotImplementedError()

    def do(self):
        # Get a task and parse it.
        r = self.application.config.services.tasks.assignOne.put(
                task=self.application.config.conf['tasks']['do'])
        if r.status_code != 200:
            return {'nb': 0, 'code': r.status_code, 'reason': r.reason, 'ok': False}

        task = r.json()
        task_key = task['_id'].split('/')[-1]
        task_data = task['data']

        try:
            logging.debug('DO : Got task: {}'.format(task_key))
            logging.debug('DO : Got task data: {}'.format(task_data))
            # Load the statuses.
            out = self.do_something(task_key, task_data)

            # Set the task as `done`.
            self.application.config.services.tasks.action.put(
                task=self.application.config.conf['tasks']['do'],
                key=escape.url_escape(task_key),
                action='success',
                data={},
                )
            return {'nb': 1, 'key': task_key, 'ok': True, 'out': out}
        except Exception as e:
            # Set the task as `fail`.
            self.application.config.services.tasks.action.put(
                task=self.application.config.conf['tasks']['do'],
                key=escape.url_escape(task_key),
                action='error',
                data={},
                )
            logging.exception('DO : Failed doing task {}.'.format(task_key))
            return {'nb': 0, 'key': task_key, 'ok': False, 'reason': e.__repr__()}

class Swagger(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Get the module documentation.",
                "method": "GET",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def initialize(self, handlers=[]):
        self.handlers = handlers

    def get(self):
        sw = OrderedDict([
            ("swaggerVersion", "1.2"),
            ("resourcePath", "/{}".format(self.application.config.conf['name'])),
            ("basePath", "/api"),
            ("apiVersion", "1.0"),
            ("produces", ["*/*", "application/json"]),
            ("apis", []),
            ])

        for h in self.application.handler_list:
            uri, handler = h[:2]
            if hasattr(handler, 'swagger'):
                sw['apis'].append(OrderedDict([
                    ('path', handler.swagger.get('path', "/{name}/{uri}").format(
                        name=self.application.config.conf['name'],
                        uri=uri.lstrip('/'))),
                    ('operations', handler.swagger.get('operations', []))
                    ]))

        self.write(json.dumps(sw, indent=2))

class Application(web.Application):
    def __init__(self, config, handlers, **kwargs):
        self.config = config if isinstance(config, Config) else Config(config)
        self.handler_list = [
            ("/swagger.json", Swagger),
            ("/swagger", web.RedirectHandler, {'url': '/swagger.json'}),
            ("/heartbeat", Heartbeat),
            ("/info", Info),
            ] + handlers
        super(Application, self).__init__(self.handler_list, **kwargs)

    def start_server(self):
        logging.basicConfig(
            level=self.config.conf['log']['level'],  # Set to 10 for debug.
            filename=self.config.conf['log']['file'],
            format='%(asctime)s (%(filename)s:%(lineno)s)- %(levelname)s - %(message)s',
            )
        logging.Formatter.converter = time.gmtime
        logging.getLogger('requests').setLevel(logging.WARNING)
        logging.getLogger('tornado').setLevel(logging.WARNING)
        logging.info('='*80)

        port = self.config.get_port()  # We need to have a fixed port in both forks.
        logging.info('Listening on port {}'.format(port))
        time.sleep(2)  # We sleep for a few seconds to let the registry start.
        if os.fork():
            self.config.register()
            server = httpserver.HTTPServer(self)
            server.bind(self.config.get_port(), address='0.0.0.0')
            server.start(self.config.conf['threads_nb'])
            ioloop.IOLoop.current().start()
        else:
            if self.config.conf.get('callbacks', None) is not None:
                for key, val in self.config.conf['callbacks'].items():
                    if os.fork():
                        if val['threads']:
                            process.fork_processes(val['threads'])
                            ioloop.PeriodicCallback(
                                Callback(self.config, val['uri'],
                                         sleep_duration=val.get('sleep', 0)),
                                val['period']*1000).start()
                            ioloop.IOLoop.instance().start()

if __name__ == '__main__':

    class MyToDo(Todo):
        def todo_loop(self, data):
            for k in range(2):
                data['nb'] += 1
                yield 'ABCDE'[data['nb'] % 5], {}

    class MyDo(Do):
        def do_something(self, task_key, task_data):
            return 'something'

    app = Application('config.yml', [
        ("/todo", MyToDo),
        ("/do", MyDo),
        ], )

    app.start_server()
<commit_msg>Move most methods of Config to Application<commit_after>import os
import time
import json
import logging
import socket
import yaml
import re
from collections import OrderedDict

import pymongo
import requests
import pandas as pd
from tornado import ioloop, web, httpserver, process, httpclient, escape

__version__ = '0.2.0'

class Kwargs(object):
    def __init__(self, **kwargs):
        for key, val in kwargs.items():
            self.__setattr__(key, val)

class WebMethod(object):
    def __init__(self, method, url):
        self.method = method
        self.url = url
        self.params = re.findall("{(.*?)}", self.url)
        self.__doc__ = (
            '\nParameters\n----------\n' +
            '\n'.join(["{} : str".format(x) for x in self.params]))

    def __call__(self, data='', **kwargs):
        response = requests.request(
            method=self.method,
            url=self.url.format(**kwargs),
            data=pd.json.dumps(data),
            )
        if not response.ok:
            logging.warning('Error in {} {}'.format(self.method, self.url.format(**kwargs)))
            raise web.HTTPError(response.status_code, response.reason)
        return response

class Config(object):
    def __init__(self, filename):
        self.conf = yaml.load(open(filename))

class Callback(object):
    def __init__(self, application, uri, sleep_duration=0):
        self.application = application
        self.uri = uri
        self.sleep_duration = sleep_duration

    def __call__(self):
        logging.debug('{} callback started'.format(self.uri))
        r = requests.post('http://localhost:{}/{}'.format(
                self.application.get_port(),
                self.uri.lstrip('/')))
        if r.status_code != 200:
            logging.debug('{} callback returned {}. Sleep for a while.'.format(
                self.uri, r.status_code))
            time.sleep(self.sleep_duration)
        logging.debug('{} callback finished : {}'.format(self.uri, r.text))

class Info(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Get the information on the service's parameters.",
                "method": "GET",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def get(self):
        self.write(self.application.config.conf)

class Heartbeat(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Tell the registry that the service is alive.",
                "method": "POST",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def post(self):
        self.application.register()
        self.write("ok")

class Todo(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Update the list of tasks to be done.",
                "method": "POST",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def post(self):
        out = self.todo()
        if out['nb'] == 0:
            self.set_status(201)  # Nothing to do.
        self.write(json.dumps(out))

    def todo_loop(self, data):
        raise NotImplementedError()

    def todo(self):
        nb_created_tasks = 0

        # We get the `todo` task.
        r = self.application.services.tasks.action.put(
            task=self.application.config.conf['tasks']['todo'],
            key=self.application.config.conf['tasks']['todo'],
            action='stack',
            data={},
            )

        logging.debug('TODO : Start scanning for new tasks')
        nb_loops = 0

        while True:
            # Get and self-assign the task.
            r = self.application.services.tasks.assignOne.put(
                    task=self.application.config.conf['tasks']['todo'])
            if r.status_code != 200:
                break

            try:
                task = r.json()

                # If `lastScanObjectId` is not defined, we start from 1970-01-01.
                data = task['data']
                data['nb'] = data.get('nb', 0)

                # Get all documents after `lastScanObjectId`
                # #########################################
                todo_tasks = list(self.todo_loop(data))
                logging.debug('TODO : Found {} tasks'.format(len(todo_tasks)))
                for task_key, task_data in todo_tasks:
                    logging.debug('TODO : Set task {}/{}'.format(task_key, task_data))
                    r = self.application.services.tasks.action.put(
                        task=self.application.config.conf['tasks']['do'],
                        key=escape.url_escape(task_key),
                        action='stack',
                        data=task_data,
                        )
                    nb_created_tasks += 1

                # Update the task to `done` if nothing happenned since last GET.
                r = self.application.services.tasks.action.put(
                    task=self.application.config.conf['tasks']['todo'],
                    key=self.application.config.conf['tasks']['todo'],
                    action='success',
                    data=data,
                    )
            except Exception as e:
                # Update the task to `done` if nothing happenned since last GET.
                r = self.application.services.tasks.action.put(
                    task=self.application.config.conf['tasks']['todo'],
                    key=self.application.config.conf['tasks']['todo'],
                    action='error',
                    data={},
                    )
                logging.exception('TODO : Failed todoing.')
                return {'nb': 0, 'ok': False, 'reason': e.__repr__()}

            nb_loops += 1

        log_str = 'TODO : Finished scanning for new tasks. Found {} in {} loops.'.format(
            nb_created_tasks, nb_loops)
        if nb_created_tasks > 0:
            logging.info(log_str)
        else:
            logging.debug(log_str)

        return {'nb': nb_created_tasks, 'nbLoops': nb_loops}

class Do(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Pick one task and do it.",
                "method": "POST",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def post(self):
        out = self.do()
        if out['nb'] == 0:
            self.set_status(201)  # Nothing to do.
        self.write(json.dumps(out))

    def do_something(self, task_key, task_data):
        raise NotImplementedError()

    def do(self):
        # Get a task and parse it.
        r = self.application.services.tasks.assignOne.put(
                task=self.application.config.conf['tasks']['do'])
        if r.status_code != 200:
            return {'nb': 0, 'code': r.status_code, 'reason': r.reason, 'ok': False}

        task = r.json()
        task_key = task['_id'].split('/')[-1]
        task_data = task['data']

        try:
            logging.debug('DO : Got task: {}'.format(task_key))
            logging.debug('DO : Got task data: {}'.format(task_data))
            # Load the statuses.
            out = self.do_something(task_key, task_data)

            # Set the task as `done`.
            self.application.services.tasks.action.put(
                task=self.application.config.conf['tasks']['do'],
                key=escape.url_escape(task_key),
                action='success',
                data={},
                )
            return {'nb': 1, 'key': task_key, 'ok': True, 'out': out}
        except Exception as e:
            # Set the task as `fail`.
            self.application.services.tasks.action.put(
                task=self.application.config.conf['tasks']['do'],
                key=escape.url_escape(task_key),
                action='error',
                data={},
                )
            logging.exception('DO : Failed doing task {}.'.format(task_key))
            return {'nb': 0, 'key': task_key, 'ok': False, 'reason': e.__repr__()}

class Swagger(web.RequestHandler):
    swagger = {
        "path": "/{name}/{uri}",
        "operations": [
            {
                "notes": "Get the module documentation.",
                "method": "GET",
                "responseMessages": [
                    {"message": "OK", "code": 200},
                    {"message": "Unauthorized", "code": 401},
                    {"message": "Forbidden", "code": 403},
                    {"message": "Not Found", "code": 404}
                    ],
                "deprecated": False,
                "produces": ["application/json"],
                "parameters": []
                }
            ]}

    def initialize(self, handlers=[]):
        self.handlers = handlers

    def get(self):
        sw = OrderedDict([
            ("swaggerVersion", "1.2"),
            ("resourcePath", "/{}".format(self.application.config.conf['name'])),
            ("basePath", "/api"),
            ("apiVersion", "1.0"),
            ("produces", ["*/*", "application/json"]),
            ("apis", []),
            ])

        for h in self.application.handler_list:
            uri, handler = h[:2]
            if hasattr(handler, 'swagger'):
                sw['apis'].append(OrderedDict([
                    ('path', handler.swagger.get('path', "/{name}/{uri}").format(
                        name=self.application.config.conf['name'],
                        uri=uri.lstrip('/'))),
                    ('operations', handler.swagger.get('operations', []))
                    ]))

        self.write(json.dumps(sw, indent=2))

class Application(web.Application):
    def __init__(self, config, handlers, **kwargs):
        self.config = config if isinstance(config, Config) else Config(config)
        self.handler_list = [
            ("/swagger.json", Swagger),
            ("/swagger", web.RedirectHandler, {'url': '/swagger.json'}),
            ("/heartbeat", Heartbeat),
            ("/info", Info),
            ] + handlers
        super(Application, self).__init__(self.handler_list, **kwargs)

        # Create mongo attribute
        self.mongo = Kwargs()
        _mongo = self.config.conf.get('db', {}).get('mongo', {})
        self.mongo = Kwargs(**{
            collname: pymongo.MongoClient(host['address'],
                                          connect=False)[db['name']][coll['name']]
            for hostname, host in _mongo.get('host', {}).items()
            for dbname, db in _mongo.get('database', {}).items() if db['host'] == hostname
            for collname, coll in _mongo.get('collection', {}).items() if coll['database'] == dbname
            })

        # Create service attribute
        self.services = Kwargs(**{
            key: Kwargs(**{
                subkey: Kwargs(**{
                    subsubkey: WebMethod(
                        subsubkey,
                        self.config.conf['registry']['url'].rstrip('/')+subsubval,
                        )
                    for subsubkey, subsubval in subval.items()})
                for subkey, subval in val.items()
                })
            for key, val in self.config.conf.get('services', {}).items()})

    def register(self):
        request = httpclient.HTTPRequest(
            '{}/register/{}'.format(
                self.config.conf['registry']['url'].rstrip('/'),
                self.config.conf['name'],
                ),
            method='POST',
            body=json.dumps({
                'url': 'http://{}:{}'.format(socket.gethostname(),
                                             self.get_port()),
                'config': self.config.conf,
                }),
            )
        client = httpclient.HTTPClient()
        r = client.fetch(request, raise_error=False)
        logging.debug('HEARTBEAT : {} ({}).'.format(
                r.code, r.reason[:30]))

    def get_port(self):
        if 'port' not in self.config.conf:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.bind(("", 0))
            self.config.conf['port'] = s.getsockname()[1]
            s.close()
        return self.config.conf['port']

    def start_server(self):
        logging.basicConfig(
            level=self.config.conf['log']['level'],  # Set to 10 for debug.
            filename=self.config.conf['log']['file'],
            format='%(asctime)s (%(filename)s:%(lineno)s)- %(levelname)s - %(message)s',
            )
        logging.Formatter.converter = time.gmtime
        logging.getLogger('requests').setLevel(logging.WARNING)
        logging.getLogger('tornado').setLevel(logging.WARNING)
        logging.info('='*80)

        port = self.get_port()  # We need to have a fixed port in both forks.
        logging.info('Listening on port {}'.format(port))
        time.sleep(2)  # We sleep for a few seconds to let the registry start.
        if os.fork():
            self.register()
            server = httpserver.HTTPServer(self)
            server.bind(self.get_port(), address='0.0.0.0')
            server.start(self.config.conf['threads_nb'])
            ioloop.IOLoop.current().start()
        else:
            if self.config.conf.get('callbacks', None) is not None:
                for key, val in self.config.conf['callbacks'].items():
                    if os.fork():
                        if val['threads']:
                            process.fork_processes(val['threads'])
                            ioloop.PeriodicCallback(
                                Callback(self, val['uri'],
                                         sleep_duration=val.get('sleep', 0)),
                                val['period']*1000).start()
                            ioloop.IOLoop.instance().start()

if __name__ == '__main__':

    class MyToDo(Todo):
        def todo_loop(self, data):
            for k in range(2):
                data['nb'] += 1
                yield 'ABCDE'[data['nb'] % 5], {}

    class MyDo(Do):
        def do_something(self, task_key, task_data):
            return 'something'

    app = Application('config.yml', [
        ("/todo", MyToDo),
        ("/do", MyDo),
        ], )

    app.start_server()
<|endoftext|>


<commit_before>"""
Copyright 2014-2016 University of Illinois

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# accounts/views.py
from __future__ import absolute_import
from django.http import HttpResponse
from django.contrib.auth import logout
from django.contrib import messages

from django.core.urlresolvers import reverse_lazy, reverse
from django.db.models import Q

from django.contrib.messages.views import SuccessMessageMixin
from django.views.generic          import TemplateView
from django.views.generic          import FormView
from django.views.generic          import RedirectView
from django.contrib.auth.mixins    import LoginRequiredMixin

from django.contrib.auth.models import User
from userProfiles.models        import UserProfile
from stats.models               import StatsUser

from django.forms.models import inlineformset_factory

from django import forms

from websiteResults.models import WebsiteReport

from timezone_field import TimeZoneFormField

from reports.views import FAENavigationMixin

from fae2.settings import SITE_URL

# Create your views here.

class ShibbolethLogout(RedirectView):

    def get_redirect_url(self, *args, **kwargs):
        logout(self.request)
        self.url = SITE_URL + '/Shibboleth.sso/Logout'
        return super(ShibbolethLogout, self).get_redirect_url(*args, **kwargs)

class ShibbolethLogin(FAENavigationMixin, TemplateView):
    template_name = 'registration/login.html'

    def get_context_data(self, **kwargs):
        context = super(ShibbolethLogin, self).get_context_data(**kwargs)

        try:
          context['user']     = self.request.user
        except:
          context['user']     = 'none'

        return context

class Logout(FAENavigationMixin, TemplateView):
    template_name = 'registration/logout.html'

    def get(self, request, *args, **kwargs):
        logout(request)
        return super(Logout, self).get(request, *args, **kwargs)

class Login(FAENavigationMixin, TemplateView):
    template_name = 'registration/login.html'

    def get_context_data(self, **kwargs):
        context = super(Login, self).get_context_data(**kwargs)

        try:
          context['user']     = self.request.user
        except:
          context['user']     = 'none'

        return context

class UserProfileForm(forms.Form):
    first_name          = forms.CharField(max_length=30)
    last_name           = forms.CharField(max_length=30)
    email               = forms.EmailField()
    org                 = forms.CharField(label="Organization", max_length=127, required=False)
    dept                = forms.CharField(label="Department", max_length=127, required=False)
    email_announcements = forms.BooleanField(required=False)
    timezone            = TimeZoneFormField()

class UpdateUserProfileView(LoginRequiredMixin, FAENavigationMixin, SuccessMessageMixin, FormView):
    template_name = 'accounts/my_account.html'
    form_class    = UserProfileForm

    success_url = reverse_lazy('user_profile')
    success_message = "Updated %(first_name)s %(last_name)s Profile"

    login_url = reverse_lazy('run_anonymous_report')
    redirect_field_name = "Anonymous Report"

    updated = False
    errors  = False

    def form_valid(self, form):

        user = self.request.user
        user.first_name = form.cleaned_data['first_name']
        user.last_name  = form.cleaned_data['last_name']
        user.email      = form.cleaned_data['email']
        user.save()

        profile           = user.profile
        profile.org       = form.cleaned_data['org']
        profile.dept      = form.cleaned_data['dept']
        profile.timezone  = form.cleaned_data['timezone']
        profile.email_announcements  = form.cleaned_data['email_announcements']
        profile.save()

        return super(UpdateUserProfileView, self).form_valid(form)

    def get_initial(self):
        # Populate ticks in BooleanFields
        user = self.request.user
        initial = {}
        initial['first_name'] = user.first_name
        initial['last_name']  = user.last_name
        initial['email']      = user.email
        initial['org']        = user.profile.org
        initial['dept']       = user.profile.dept
        initial['timezone']   = user.profile.timezone
        initial['email_announcements']        = user.profile.email_announcements
        return initial

    def get_context_data(self, **kwargs):
        context = super(UpdateUserProfileView, self).get_context_data(**kwargs)

        context['user_stats'] = StatsUser.objects.get(user=self.request.user)
        context['user_profile'] = UserProfile.objects.get(user=self.request.user)

        return context

# ==============================================================
#
# Administration Status View
#
# ==============================================================

class StatusView(LoginRequiredMixin, FAENavigationMixin, TemplateView):
    template_name = 'accounts/status.html'

    def get_context_data(self, **kwargs):
        context = super(StatusView, self).get_context_data(**kwargs)

        reports = WebsiteReport.objects.all()

        context['initialized'] = reports.filter(Q(status='-') | Q(status='I'))
        context['processing']  = reports.filter(Q(status='A') | Q(status='S'))
        context['errors']      = reports.filter(status='E')

        return context

# ==============================================================
#
# Administration Status View Use Information View
#
# ==============================================================

class AllUserInformationView(LoginRequiredMixin, FAENavigationMixin, TemplateView):
    template_name = 'accounts/all_user_information.html'

    def get_context_data(self, **kwargs):
        context = super(AllUserInformationView, self).get_context_data(**kwargs)

        user_profiles = UserProfile.objects.all()

        context['include_announcements']  = user_profiles.filter(email_announcements=True)
        context['exclude_announcements']  = user_profiles.filter(email_announcements=False)
        context['stats_users']             = StatsUser.objects.all()

        return context

class UserInformationView(LoginRequiredMixin, FAENavigationMixin, TemplateView):
    template_name = 'accounts/user_information.html'

    def get_context_data(self, **kwargs):
        context = super(UserInformationView, self).get_context_data(**kwargs)

        user = User.objects.get(id=kwargs['user_id'])
        user_profile = UserProfile.objects.get(user=user)
        stats_user   = StatsUser.objects.get(user=user)

        context['user_profile'] = user_profile
        context['stats_user']   = stats_user

        return context

<commit_msg>shib debug<commit_after>"""
Copyright 2014-2016 University of Illinois

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# accounts/views.py
from __future__ import absolute_import
from django.http import HttpResponse
from django.contrib.auth import logout
from django.contrib import messages

from django.core.urlresolvers import reverse_lazy, reverse
from django.db.models import Q

from django.contrib.messages.views import SuccessMessageMixin
from django.views.generic          import TemplateView
from django.views.generic          import FormView
from django.views.generic          import RedirectView
from django.contrib.auth.mixins    import LoginRequiredMixin

from django.contrib.auth.models import User
from userProfiles.models        import UserProfile
from stats.models               import StatsUser

from django.forms.models import inlineformset_factory

from django import forms

from websiteResults.models import WebsiteReport

from timezone_field import TimeZoneFormField

from reports.views import FAENavigationMixin

from fae2.settings import SITE_URL

# Create your views here.

class ShibbolethLogout(RedirectView):

    def get_redirect_url(self, *args, **kwargs):
        logout(self.request)
        self.url = SITE_URL + '/Shibboleth.sso/Logout'
        return super(ShibbolethLogout, self).get_redirect_url(*args, **kwargs)

class ShibbolethLogin(RedirectView):

    def get_redirect_url(self, *args, **kwargs):
        self.url = SITE_URL
        return super(ShibbolethLogin, self).get_redirect_url(*args, **kwargs)

class Logout(FAENavigationMixin, TemplateView):
    template_name = 'registration/logout.html'

    def get(self, request, *args, **kwargs):
        logout(request)
        return super(Logout, self).get(request, *args, **kwargs)

class Login(FAENavigationMixin, TemplateView):
    template_name = 'registration/login.html'

    def get_context_data(self, **kwargs):
        context = super(Login, self).get_context_data(**kwargs)

        try:
          context['user']     = self.request.user
        except:
          context['user']     = 'none'

        return context

class UserProfileForm(forms.Form):
    first_name          = forms.CharField(max_length=30)
    last_name           = forms.CharField(max_length=30)
    email               = forms.EmailField()
    org                 = forms.CharField(label="Organization", max_length=127, required=False)
    dept                = forms.CharField(label="Department", max_length=127, required=False)
    email_announcements = forms.BooleanField(required=False)
    timezone            = TimeZoneFormField()

class UpdateUserProfileView(LoginRequiredMixin, FAENavigationMixin, SuccessMessageMixin, FormView):
    template_name = 'accounts/my_account.html'
    form_class    = UserProfileForm

    success_url = reverse_lazy('user_profile')
    success_message = "Updated %(first_name)s %(last_name)s Profile"

    login_url = reverse_lazy('run_anonymous_report')
    redirect_field_name = "Anonymous Report"

    updated = False
    errors  = False

    def form_valid(self, form):

        user = self.request.user
        user.first_name = form.cleaned_data['first_name']
        user.last_name  = form.cleaned_data['last_name']
        user.email      = form.cleaned_data['email']
        user.save()

        profile           = user.profile
        profile.org       = form.cleaned_data['org']
        profile.dept      = form.cleaned_data['dept']
        profile.timezone  = form.cleaned_data['timezone']
        profile.email_announcements  = form.cleaned_data['email_announcements']
        profile.save()

        return super(UpdateUserProfileView, self).form_valid(form)

    def get_initial(self):
        # Populate ticks in BooleanFields
        user = self.request.user
        initial = {}
        initial['first_name'] = user.first_name
        initial['last_name']  = user.last_name
        initial['email']      = user.email
        initial['org']        = user.profile.org
        initial['dept']       = user.profile.dept
        initial['timezone']   = user.profile.timezone
        initial['email_announcements']        = user.profile.email_announcements
        return initial

    def get_context_data(self, **kwargs):
        context = super(UpdateUserProfileView, self).get_context_data(**kwargs)

        context['user_stats'] = StatsUser.objects.get(user=self.request.user)
        context['user_profile'] = UserProfile.objects.get(user=self.request.user)

        return context

# ==============================================================
#
# Administration Status View
#
# ==============================================================

class StatusView(LoginRequiredMixin, FAENavigationMixin, TemplateView):
    template_name = 'accounts/status.html'

    def get_context_data(self, **kwargs):
        context = super(StatusView, self).get_context_data(**kwargs)

        reports = WebsiteReport.objects.all()

        context['initialized'] = reports.filter(Q(status='-') | Q(status='I'))
        context['processing']  = reports.filter(Q(status='A') | Q(status='S'))
        context['errors']      = reports.filter(status='E')

        return context

# ==============================================================
#
# Administration Status View Use Information View
#
# ==============================================================

class AllUserInformationView(LoginRequiredMixin, FAENavigationMixin, TemplateView):
    template_name = 'accounts/all_user_information.html'

    def get_context_data(self, **kwargs):
        context = super(AllUserInformationView, self).get_context_data(**kwargs)

        user_profiles = UserProfile.objects.all()

        context['include_announcements']  = user_profiles.filter(email_announcements=True)
        context['exclude_announcements']  = user_profiles.filter(email_announcements=False)
        context['stats_users']             = StatsUser.objects.all()

        return context

class UserInformationView(LoginRequiredMixin, FAENavigationMixin, TemplateView):
    template_name = 'accounts/user_information.html'

    def get_context_data(self, **kwargs):
        context = super(UserInformationView, self).get_context_data(**kwargs)

        user = User.objects.get(id=kwargs['user_id'])
        user_profile = UserProfile.objects.get(user=user)
        stats_user   = StatsUser.objects.get(user=user)

        context['user_profile'] = user_profile
        context['stats_user']   = stats_user

        return context

<|endoftext|>