<commit_before>'''
Implementation of genome graphs.

Data structure :
g.nodes : dict to store node attributes (seq, name)

g.edges : adjacency list of the nodes.
       A negative nodeId indicates that the contig is reversed. In that case,
       the source is the start of the contig, and the target is the end.

'''

import re
from utils import reverse_complement
from SequenceAlignment import NeedlemanWunsch

class GenomeNode:

       def __init__(self,nodeSeq,nodeName):
              # self.nodeId = nodeId
              self.nodeSeq = nodeSeq
              self.nodeName = nodeName
              # self.active = True  # Alternative way to remove nodes ?

       def __eq__(self, other):
              if isinstance(other, GenomeNode):
                     return self.nodeSeq == other.nodeSeq and self.nodeName == other.nodeName
              return False

       def __ne__(self, other):
              return not self.__eq__(other)

       def __hash__(self):
              return hash((self.nodeSeq, self.nodeName))

class GenomeGraph:

       def __init__(self):
              self.nodes = {}
              self.edges = {}
              self.overlap = 0

       def nNodes(self):
              return(len(self.nodes))

       def nEdges(self):
              return(len([j for i in self.edges.values() for j in i])/2)

       def add_node(self,nodeName,nodeSeq):
              newNode = GenomeNode(nodeSeq,nodeName)
              try:
                     assert newNode not in self.nodes.values()
              except AssertionError:
                     print("Node already in graph")

              if len(self.nodes)>0:
                     nodeId = max(self.nodes.keys())+1
              else:
                     nodeId = 1
              self.nodes[nodeId] = newNode
              self.edges[nodeId] = set()
              self.edges[-nodeId] = set()

       def add_edge(self, src,dst):
              self.edges[src].add(dst)
              self.edges[-dst].add(-src)

       def rem_edge(self,src,dst):
              try:
                     self.edges[src].remove(dst)
                     self.edges[-dst].remove(-src)
              except KeyError:
                     print("Edge not in graph")

       def rem_node(self,nodeId):
              try:
                     self.nodes.pop(nodeId)
              except KeyError:
                     print("Node not in graph")

              for n in self.get_neighbors(nodeId).copy():
                     self.rem_edge(-n,-nodeId)

              for n in self.get_neighbors(-nodeId).copy():
                     self.rem_edge(-n,nodeId)

              self.edges.pop(nodeId)
              self.edges.pop(-nodeId)

       def get_neighbors(self,nodeId):
              return(self.edges[nodeId])

       def get_node_seq(self,nodeId):
              if nodeId < 0:
                     nodeSeq = reverse_complement(self.nodes[-nodeId].nodeSeq.strip())
              else:
                     nodeSeq = self.nodes[nodeId].nodeSeq.strip()
              return(nodeSeq)

       @classmethod
       def read_gfa(self,file):

              g = GenomeGraph()
              nodeIds = {}  # Used to retrieve a node Id from a name. Is it useful though?
              nlines = 0
              with open(file) as f:
                     for line in f:
                            nlines += 1
                            if re.match(r"S.*", line):
                                   nodeName = line.split("\t")[1]
                                   nodeSeq = line.split("\t")[2]
                                   g.add_node(nodeName,nodeSeq)
                                   nodeIds[nodeName] = g.nNodes()
                            elif re.match(r"L.*",line):
                                   startName,startDir,endName,endDir,overlap = line.split("\t")[1:]

                                   if g.overlap == 0:
                                          g.overlap = overlap

                                   startNode = nodeIds[startName]
                                   endNode = nodeIds[endName]

                                   if startDir == "-":
                                          startNode = - startNode
                                   if endDir == "-":
                                          endNode = - endNode

                                   g.add_edge(startNode,endNode)
              return(g)

       def write_gfa(self,filename):
              with open(filename,"w") as f:
                     for node in self.nodes.values():
                            f.write("S\t"+node.nodeName+"\t"+node.nodeSeq)
                     written_edges = set()
                     for src_id in self.nodes.keys():
                            src_name = self.nodes[abs(src_id)].nodeName
                            for dst_id in self.edges[src_id]:
                                   if (src_id,dst_id) not in written_edges:
                                          dst_name = self.nodes[abs(dst_id)].nodeName
                                          if dst_id > 0:
                                                 f.write("L\t"+src_name+"\t+\t"+dst_name+"\t+\t"+self.overlap)
                                          else:
                                                 f.write("L\t"+src_name+"\t+\t"+dst_name+"\t-\t"+self.overlap)
                                          written_edges.add((src_id,dst_id))
                                          written_edges.add((-dst_id,-src_id))
                            for dst_id in self.edges[-src_id]:
                                   if (-src_id,dst_id) not in written_edges:
                                          dst_name = self.nodes[abs(dst_id)].nodeName
                                          if dst_id > 0:
                                                 f.write("L\t"+src_name+"\t-\t"+dst_name+"\t+\t"+self.overlap)
                                          else:
                                                 f.write("L\t"+src_name+"\t-\t"+dst_name+"\t-\t"+self.overlap)
                                          written_edges.add((-src_id,dst_id))
                                          written_edges.add((-dst_id,src_id))

       ###########  Graph simplification ###########

       def pop_bubble(self,nodeId):
              n1 = self.get_neighbors(nodeId).copy()
              n2 = self.get_neighbors(-nodeId).copy()

              if len(n1)==len(n2)==1:
                     r = self.get_neighbors(-n1.pop())
                     l = self.get_neighbors(-n2.pop())
                     assert -nodeId in r and nodeId in l

                     r_rev = {-i for i in r}

                     inter = l & r_rev
                     assert nodeId in inter

                     if len(inter)>0:
                            toRemove = self.compare_nodes(inter)
                            for node in toRemove:
                                   self.rem_node(abs(node))

       def pop_all_bubbles(self):
              for node in list(self.nodes):
                     if node in self.nodes.keys():
                            self.pop_bubble(node)

       def compare_nodes(self,nodeSet):
              uniq = set()
              remove = set()
              for node in nodeSet:
                     nodeSeq = self.get_node_seq(node) # Gets rc if node<0

                     if node in uniq:
                            continue
                     foundmatch = False
                     for refNode in uniq:
                            refSeq =  self.get_node_seq(refNode)
                            if refSeq == nodeSeq:
                                   remove.add(node)
                                   foundmatch = True
                            else:
                                   nw = NeedlemanWunsch(refSeq, nodeSeq, 10, -5, -5)
                                   id = nw.getIdentity()
                                   if id > 0.9:
                                          remove.add(node)
                                          foundmatch = True
                     if not foundmatch:
                            uniq.add(node)
              return(remove)
<commit_msg>Small changes/bugfixes<commit_after>'''
Implementation of genome graphs.

Data structure :
g.nodes : dict to store node attributes (seq, name)

g.edges : adjacency list of the nodes.
       A negative nodeId indicates that the contig is reversed. In that case,
       the source is the start of the contig, and the target is the end.

'''

import re
from pipeline.genome_graph.utils import reverse_complement
from pipeline.genome_graph.SequenceAlignment import NeedlemanWunsch

class GenomeNode:

       def __init__(self,nodeSeq,nodeName):
              # self.nodeId = nodeId
              self.nodeSeq = nodeSeq
              self.nodeName = nodeName
              # self.active = True  # Alternative way to remove nodes ?

       def __eq__(self, other):
              if isinstance(other, GenomeNode):
                     return self.nodeSeq == other.nodeSeq and self.nodeName == other.nodeName
              return False

       def __ne__(self, other):
              return not self.__eq__(other)

       def __hash__(self):
              return hash((self.nodeSeq, self.nodeName))

class GenomeGraph:

       def __init__(self):
              self.nodes = {}
              self.edges = {}
              self.overlap = 0

       def nNodes(self):
              return(len(self.nodes))

       def nEdges(self):
              return(len([j for i in self.edges.values() for j in i])/2)

       def add_node(self,nodeName,nodeSeq):
              newNode = GenomeNode(nodeSeq,nodeName)
              try:
                     assert newNode not in self.nodes.values()
              except AssertionError:
                     print("Node already in graph")

              if len(self.nodes)>0:
                     nodeId = max(self.nodes.keys())+1
              else:
                     nodeId = 1
              self.nodes[nodeId] = newNode
              self.edges[nodeId] = set()
              self.edges[-nodeId] = set()

       def add_edge(self,n1,n2):
              self.edges[n1].add(n2)
              self.edges[-n2].add(-n1)

       def rem_edge(self,n1,n2):
              try:
                     self.edges[n1].remove(n2)
                     self.edges[-n2].remove(-n1)
              except KeyError:
                     print("Edge not in graph")

       def rem_node(self,nodeId):
              try:
                     self.nodes.pop(nodeId)
              except KeyError:
                     print("Node not in graph")

              for n in self.get_neighbors(nodeId).copy():
                     self.rem_edge(-n,-nodeId)

              for n in self.get_neighbors(-nodeId).copy():
                     self.rem_edge(-n,nodeId)

              self.edges.pop(nodeId)
              self.edges.pop(-nodeId)

       def get_neighbors(self,nodeId):
              return(self.edges[nodeId])

       def get_node_seq(self,nodeId):
              if nodeId < 0:
                     nodeSeq = reverse_complement(self.nodes[-nodeId].nodeSeq.strip())
              else:
                     nodeSeq = self.nodes[nodeId].nodeSeq.strip()
              return(nodeSeq)

       @classmethod
       def read_gfa(self,file):

              g = GenomeGraph()
              nodeIds = {}  # Used to retrieve a node Id from a name. Is it useful though?
              nlines = 0
              with open(file) as f:
                     for line in f:
                            nlines += 1
                            if re.match(r"S.*", line):
                                   nodeName = line.split("\t")[1]
                                   nodeSeq = line.split("\t")[2].strip()
                                   g.add_node(nodeName,nodeSeq)
                                   nodeIds[nodeName] = g.nNodes()
                            elif re.match(r"L.*",line):
                                   startName,startDir,endName,endDir,overlap = line.split("\t")[1:]

                                   if g.overlap == 0:
                                          overlap = int(overlap.replace('M\n',''))
                                          g.overlap = overlap

                                   startNode = nodeIds[startName]
                                   endNode = nodeIds[endName]

                                   if startDir == "-":
                                          startNode = - startNode
                                   if endDir == "-":
                                          endNode = - endNode

                                   g.add_edge(startNode,endNode)
              return(g)

       def write_gfa(self,filename):
              overlap = str(self.overlap) + "M"
              with open(filename,"w") as f:
                     for node in self.nodes.values():
                            f.write("S\t"+node.nodeName+"\t"+node.nodeSeq+"\n")
                     written_edges = set()
                     for src_id in self.nodes.keys():
                            src_name = self.nodes[abs(src_id)].nodeName
                            for dst_id in self.edges[src_id]:
                                   if (src_id,dst_id) not in written_edges:
                                          dst_name = self.nodes[abs(dst_id)].nodeName
                                          if dst_id > 0:
                                                 f.write("L\t"+src_name+"\t+\t"+dst_name+"\t+\t"+overlap+"\n")
                                          else:
                                                 f.write("L\t"+src_name+"\t+\t"+dst_name+"\t-\t"+overlap+"\n")
                                          written_edges.add((src_id,dst_id))
                                          written_edges.add((-dst_id,-src_id))
                            for dst_id in self.edges[-src_id]:
                                   if (-src_id,dst_id) not in written_edges:
                                          dst_name = self.nodes[abs(dst_id)].nodeName
                                          if dst_id > 0:
                                                 f.write("L\t"+src_name+"\t-\t"+dst_name+"\t+\t"+overlap+"\n")
                                          else:
                                                 f.write("L\t"+src_name+"\t-\t"+dst_name+"\t-\t"+overlap+"\n")
                                          written_edges.add((-src_id,dst_id))
                                          written_edges.add((-dst_id,src_id))

       ###########  Graph simplification ###########

       def pop_bubble(self,nodeId):
              n1 = self.get_neighbors(nodeId).copy()
              n2 = self.get_neighbors(-nodeId).copy()

              if len(n1)==len(n2)==1:
                     r = self.get_neighbors(-n1.pop())
                     l = self.get_neighbors(-n2.pop())
                     assert -nodeId in r and nodeId in l

                     r_rev = {-i for i in r}

                     inter = l & r_rev
                     assert nodeId in inter

                     if len(inter)>0:
                            toRemove = self.compare_nodes(inter)
                            for node in toRemove:
                                   self.rem_node(abs(node))

       def pop_all_bubbles(self):
              for node in list(self.nodes):
                     if node in self.nodes.keys():
                            self.pop_bubble(node)

       def compare_nodes(self,nodeSet):
              uniq = set()
              remove = set()
              for node in nodeSet:
                     nodeSeq = self.get_node_seq(node) # Gets rc if node<0

                     if node in uniq:
                            continue
                     foundmatch = False
                     for refNode in uniq:
                            refSeq =  self.get_node_seq(refNode)
                            if refSeq == nodeSeq:
                                   remove.add(node)
                                   foundmatch = True
                            else:
                                   nw = NeedlemanWunsch(refSeq, nodeSeq, 10, -5, -5)
                                   id = nw.getIdentity()
                                   if id > 0.9:
                                          remove.add(node)
                                          foundmatch = True
                     if not foundmatch:
                            uniq.add(node)
              return(remove)
<|endoftext|>


<commit_before>from passlib.context import CryptContext

from flask_jwt import JWT

from flask_praetorian.exceptions import PraetorianError

class Praetorian:
    # TODO: Figure out how to register the /auth endpoint with swagger

    def __init__(self, app=None, user_class=None):

        self.pwd_ctx = None
        self.hash_scheme = None
        self.salt = None

        if app is not None and user_class is not None:
            self.init_app(app, user_class)

    def init_app(self, app, user_class):
        PraetorianError.require_condition(
            app.config.get('SECRET_KEY') is not None,
            "There must be a SECRET_KEY app config setting set",
        )
        # TODO: add error handler stuff

        possible_schemes = [
            # TODO: Add argon2 support when passlib 1.7 is released (soon)
            #       https://bitbucket.org/ecollins/passlib/wiki/Home
            #       at that time, argon2 should be come the default
            # 'argon2',
            'bcrypt',
            'pbkdf2_sha512',
        ]
        # TODO: make sure to add test for a few non plaintext hash methods
        self.pwd_ctx = CryptContext(
            default='bcrypt',
            schemes=possible_schemes + ['plaintext'],
            deprecated=[],
        )

        self.hash_scheme = app.config.get('PRAETORIAN_HASH_SCHEME')

        self.user_class = user_class
        # TODO: let jwt be passed in as a parameter and optionally init_app
        self.jwt = JWT(
            app,
            authentication_handler=self._authenticate,
            identity_handler=self._identity,
        )

        app.errorhandler(PraetorianError)(self.error_handler)

        # TODO: figure out extension nameing convention
        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['jwt_roles'] = self

    def _authenticate(self, username, password):
        user = self.user_class.query.filter_by(username=username).one_or_none()
        if user is None or not self.verify_password(password, user.password):
            return None
        return user

    def _identity(self, payload):
        user_id = payload['identity']
        return self.user_class.query.get(user_id)

    def verify_password(self, raw_password, hashed_password):
        return self.pwd_ctx.verify(raw_password, hashed_password)

    def encrypt_password(self, raw_password):
        return self.pwd_ctx.encrypt(raw_password, scheme=self.hash_scheme)

    def error_handler(self, error):
        return error.jsonify(), error.status_code, error.headers
<commit_msg>Converted some TODO comments into issues<commit_after>from passlib.context import CryptContext

from flask_jwt import JWT

from flask_praetorian.exceptions import PraetorianError

class Praetorian:

    def __init__(self, app=None, user_class=None):

        self.pwd_ctx = None
        self.hash_scheme = None
        self.salt = None

        if app is not None and user_class is not None:
            self.init_app(app, user_class)

    def init_app(self, app, user_class):
        PraetorianError.require_condition(
            app.config.get('SECRET_KEY') is not None,
            "There must be a SECRET_KEY app config setting set",
        )

        possible_schemes = [
            # 'argon2',
            'bcrypt',
            'pbkdf2_sha512',
        ]
        self.pwd_ctx = CryptContext(
            default='bcrypt',
            schemes=possible_schemes + ['plaintext'],
            deprecated=[],
        )

        self.hash_scheme = app.config.get('PRAETORIAN_HASH_SCHEME')

        self.user_class = user_class
        self.jwt = JWT(
            app,
            authentication_handler=self._authenticate,
            identity_handler=self._identity,
        )

        app.errorhandler(PraetorianError)(self.error_handler)

        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['jwt_roles'] = self

    def _authenticate(self, username, password):
        user = self.user_class.query.filter_by(username=username).one_or_none()
        if user is None or not self.verify_password(password, user.password):
            return None
        return user

    def _identity(self, payload):
        user_id = payload['identity']
        return self.user_class.query.get(user_id)

    def verify_password(self, raw_password, hashed_password):
        return self.pwd_ctx.verify(raw_password, hashed_password)

    def encrypt_password(self, raw_password):
        return self.pwd_ctx.encrypt(raw_password, scheme=self.hash_scheme)

    def error_handler(self, error):
        return error.jsonify(), error.status_code, error.headers
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import yaml
from fabric.api import task, run, sudo, put, task, \
        parallel, execute, env
from cuisine import file_exists, file_write, file_append, \
        text_strip_margin, mode_sudo, file_update, ssh_keygen, \
        ssh_authorize, dir_ensure

@task
def status():
    """ Check the status """
    yml_path = __file__.replace('fabfile','ymlfile').rstrip(r'\py|\pyc') + 'yml'
    f = open(yml_path)
    cfg = yaml.safe_load(f)
    f.close()

    env.user = cfg['admin_user']
    env.disable_known_hosts = True

    hosts = []
    for host in cfg['hosts']:
        hosts.append(cfg['hosts'][host]['ipaddr'])

    execute(check_status, hosts=hosts)

def check_status():

    sudo('jps', user='hdfs')

@task
def install():
    """ Install Hadoop Cluster """
    yml_path = __file__.replace('fabfile','ymlfile').rstrip(r'\py|\pyc') + 'yml'
    f = open(yml_path)
    cfg = yaml.safe_load(f)
    f.close()

    env.user = cfg['admin_user']
    env.disable_known_hosts = True

    hosts = []
    for host in cfg['hosts']:
        hosts.append(cfg['hosts'][host]['ipaddr'])

    execute(pkg_install,hosts=hosts)
    execute(update_etc_hosts,cfg_hosts=cfg['hosts'],hosts=hosts)
    execute(update_roles,cfg_hosts=cfg['hosts'],hosts=hosts)
    sites = ['core-site',
             'hdfs-site',
             'mapred-site']
    for site in sites:
        execute(update_config,cfg_name=site,cfg_list=cfg[site],hosts=hosts)
    execute(update_env_sh,hosts=hosts)
    admin_node = cfg['admin_node']
    admin_node_ip = cfg['hosts'][admin_node]['ipaddr']
    output = execute(create_hdfs_sshkey,hosts=[admin_node_ip])
    key = output[admin_node_ip]
    execute(update_authorized_keys,key=key,hosts=hosts)
    execute(update_dir,cfg['update_dir_list'],hosts=hosts)

@parallel
def update_dir(update_dir_list):

    with mode_sudo():
        for dir in update_dir_list:
            owner = update_dir_list[dir]['owner']
            mode = update_dir_list[dir]['mode']
            dir_ensure(dir, mode=mode, owner=owner)

@parallel
def update_authorized_keys(key):
    with mode_sudo():
        ssh_authorize(user='hdfs',key=key)

def create_hdfs_sshkey():
    with mode_sudo():
        ssh_keygen(user='hdfs',keytype='rsa')
        key = sudo('cat /usr/lib/hadoop/.ssh/id_rsa.pub')
    return key

@parallel
def update_env_sh():

    file = '/usr/lib/hadoop/conf/hadoop-env.sh'
    with mode_sudo():
        file_update(file, _sub_update_env_sh)

def _sub_update_env_sh(text):
    """ Update /usr/lib/hadoop/conf/hadoop-env.sh"""
    res = []
    for line in text.split('\n'):
        if line.strip().startswith("# export JAVA_HOME"):
            res.append("export JAVA_HOME=/usr/lib/jvm/java-7-oracle")
        else:
            res.append(line)
    return '\n'.join(res) + '\n'

@parallel
def update_config(cfg_name, cfg_list):
    """ Update xml files """

    lines = []
    header = text_strip_margin(
            """
            |<?xml version="1.0"?>
            |<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
            |
            |<!-- Put site-specific property overrides in this file. -->
            |
            |<configuration>
            |""")
    lines.append(header)

    for entry in cfg_list:
        property = text_strip_margin(
                """
                |  <property>
                |    <name>{0}</name>
                |    <value>{1}</value>
                |  </property>
                |""".format(entry, cfg_list[entry]))
        lines.append(property)

    footer = '</configuration>\n'
    lines.append(footer)

    file = '/usr/lib/hadoop/conf/' + cfg_name + '.xml'
    text = '\n'.join(lines) + '\n'
    file_write(file, text, sudo=True)

@parallel
def update_etc_hosts(cfg_hosts):
    """Update /etc/hosts """

    file = '/etc/hosts'
    lines = []
    lines.append("127.0.0.1 localhost")
    for host in cfg_hosts:
        lines.append("{0} {1}".format(cfg_hosts[host]['ipaddr'], host))
    text = '\n'.join(lines) + '\n'
    file_write(file, text, sudo=True)

@parallel
def update_roles(cfg_hosts):
    """ Update /usr/lib/hadoop/conf/[masters/slaves] """

    dir = '/usr/lib/hadoop/conf/'
    masters = []
    slaves = []
    for host in cfg_hosts:
        if cfg_hosts[host]['group'] == 'masters':
            masters.append(host)
        elif cfg_hosts[host]['group'] == 'slaves':
            slaves.append(host)
    # Update masters
    file = dir + 'masters'
    text = '\n'.join(masters) + '\n'
    file_write(file, text, sudo=True)
    # Update slaves
    file = dir + 'slaves'
    text = '\n'.join(slaves) + '\n'
    file_write(file, text, sudo=True)

@parallel
def pkg_install():
    ''':hostname - Install Hadoop Master'''
    file_name = '/usr/lib/jvm/java-7-oracle'
    if not file_exists(file_name):
        sudo('add-apt-repository -y ppa:webupd8team/java')
        sudo('add-apt-repository -y ppa:hadoop-ubuntu/stable')
        sudo('apt-get update && sudo apt-get -y upgrade')
        sudo('echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections')
        sudo('echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections')
        sudo('apt-get -y install oracle-java7-installer')
        sudo('apt-get -y install hadoop')
    else:
        print '{0} exists. Oracle Java is already installed.'.format(file_name)

@task
@parallel
def enable_root_login():

    sudo('cat .ssh/authorized_keys > /root/.ssh/authorized_keys')

@parallel
def hello():

    run('hostname && id && echo hello')

<commit_msg>update fabfile/hadoop.py<commit_after>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import os
import yaml
from fabric.api import task, run, sudo, put, task, \
        parallel, execute, env
from cuisine import file_exists, file_write, file_append, \
        text_strip_margin, mode_sudo, file_update, ssh_keygen, \
        ssh_authorize, dir_ensure

@task
def status():
    """ Check the status """
    # Read cofiguration file to cfg
    cfg_dir = os.path.dirname(__file__).replace('fabfile','ymlfile')
    cfg_file = cfg_dir + '/hadoop.yml'
    f = open(cfg_file)
    cfg = yaml.safe_load(f)
    f.close()

    # Set ssh user and have ssh not check .ssh/known_hosts
    env.user = cfg['admin_user']
    env.disable_known_hosts = True

    # Set hosts
    hosts = []
    for host in cfg['hosts']:
        hosts.append(cfg['hosts'][host]['ipaddr'])

    # Execute check_status on the hosts.
    execute(check_status, hosts=hosts)

def check_status():

    sudo('jps', user='hdfs')

@task
def install():
    """ Install Hadoop Cluster """
    cfg_dir = os.path.dirname(__file__).replace('fabfile','ymlfile')
    cfg_file = cfg_dir + '/hadoop.yml'
    f = open(cfg_file)
    cfg = yaml.safe_load(f)
    f.close()

    env.user = cfg['admin_user']
    env.disable_known_hosts = True

    hosts = []
    for host in cfg['hosts']:
        hosts.append(cfg['hosts'][host]['ipaddr'])

    execute(pkg_install,hosts=hosts)
    execute(update_etc_hosts,cfg_hosts=cfg['hosts'],hosts=hosts)
    execute(update_roles,cfg_hosts=cfg['hosts'],hosts=hosts)
    sites = ['core-site',
             'hdfs-site',
             'mapred-site']
    for site in sites:
        execute(update_config,cfg_name=site,cfg_list=cfg[site],hosts=hosts)
    execute(update_env_sh,hosts=hosts)
    admin_node = cfg['admin_node']
    admin_node_ip = cfg['hosts'][admin_node]['ipaddr']
    output = execute(create_hdfs_sshkey,hosts=[admin_node_ip])
    key = output[admin_node_ip]
    execute(update_authorized_keys,key=key,hosts=hosts)
    execute(update_dir,cfg['update_dir_list'],hosts=hosts)

@parallel
def update_dir(update_dir_list):

    with mode_sudo():
        for dir in update_dir_list:
            owner = update_dir_list[dir]['owner']
            mode = update_dir_list[dir]['mode']
            dir_ensure(dir, mode=mode, owner=owner)

@parallel
def update_authorized_keys(key):
    with mode_sudo():
        ssh_authorize(user='hdfs',key=key)

def create_hdfs_sshkey():
    with mode_sudo():
        ssh_keygen(user='hdfs',keytype='rsa')
        key = sudo('cat /usr/lib/hadoop/.ssh/id_rsa.pub')
    return key

@parallel
def update_env_sh():

    file = '/usr/lib/hadoop/conf/hadoop-env.sh'
    with mode_sudo():
        file_update(file, _sub_update_env_sh)

def _sub_update_env_sh(text):
    """ Update /usr/lib/hadoop/conf/hadoop-env.sh"""
    res = []
    for line in text.split('\n'):
        if line.strip().startswith("# export JAVA_HOME"):
            res.append("export JAVA_HOME=/usr/lib/jvm/java-7-oracle")
        else:
            res.append(line)
    return '\n'.join(res) + '\n'

@parallel
def update_config(cfg_name, cfg_list):
    """ Update xml files """

    lines = []
    header = text_strip_margin(
            """
            |<?xml version="1.0"?>
            |<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
            |
            |<!-- Put site-specific property overrides in this file. -->
            |
            |<configuration>
            |""")
    lines.append(header)

    for entry in cfg_list:
        property = text_strip_margin(
                """
                |  <property>
                |    <name>{0}</name>
                |    <value>{1}</value>
                |  </property>
                |""".format(entry, cfg_list[entry]))
        lines.append(property)

    footer = '</configuration>\n'
    lines.append(footer)

    file = '/usr/lib/hadoop/conf/' + cfg_name + '.xml'
    text = '\n'.join(lines) + '\n'
    file_write(file, text, sudo=True)

@parallel
def update_etc_hosts(cfg_hosts):
    """Update /etc/hosts """

    file = '/etc/hosts'
    lines = []
    lines.append("127.0.0.1 localhost")
    for host in cfg_hosts:
        lines.append("{0} {1}".format(cfg_hosts[host]['ipaddr'], host))
    text = '\n'.join(lines) + '\n'
    file_write(file, text, sudo=True)

@parallel
def update_roles(cfg_hosts):
    """ Update /usr/lib/hadoop/conf/[masters/slaves] """

    dir = '/usr/lib/hadoop/conf/'
    masters = []
    slaves = []
    for host in cfg_hosts:
        if cfg_hosts[host]['group'] == 'masters':
            masters.append(host)
        elif cfg_hosts[host]['group'] == 'slaves':
            slaves.append(host)
    # Update masters
    file = dir + 'masters'
    text = '\n'.join(masters) + '\n'
    file_write(file, text, sudo=True)
    # Update slaves
    file = dir + 'slaves'
    text = '\n'.join(slaves) + '\n'
    file_write(file, text, sudo=True)

@parallel
def pkg_install():
    ''':hostname - Install Hadoop Master'''
    file_name = '/usr/lib/jvm/java-7-oracle'
    if not file_exists(file_name):
        sudo('add-apt-repository -y ppa:webupd8team/java')
        sudo('add-apt-repository -y ppa:hadoop-ubuntu/stable')
        sudo('apt-get update && sudo apt-get -y upgrade')
        sudo('echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections')
        sudo('echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections')
        sudo('apt-get -y install oracle-java7-installer')
        sudo('apt-get -y install hadoop')
    else:
        print '{0} exists. Oracle Java is already installed.'.format(file_name)

@task
@parallel
def enable_root_login():

    sudo('cat .ssh/authorized_keys > /root/.ssh/authorized_keys')

@parallel
def hello():

    run('hostname && id && echo hello')

<|endoftext|>


<commit_before>"""
BottlePy API Implementation for ISWQ2014
"""
import logging

logging.basicConfig(format='%(asctime)s %(message)s')

LOGGER = logging.getLogger('app')
LOGGER.setLevel('INFO')

import json
from bottle import *
from models.Movie import *
from templates.LinkCJ import LinkCJ
from templates.ErrorCJ import ErrorCJ
from templates.MoviesTemplateCJ import MoviesTemplateCJ
from templates.MovieItemCJ import MovieItemCJ
from templates.MoviesCollectionTemplateCJ import MoviesCollectionTemplateCJ
import mongoengine
import datetime

HOST = 'localhost'
PORT = 1337
PATH = '/api/'
BASE = 'http://'
ROOT = BASE + HOST + ':' + str(PORT) + PATH
DATETIMESTAMP = datetime.datetime.utcnow().isoformat() + 'Z'

# application setting
app = default_app();
app.config['autojson'] = False

# setup response Content-Type
def determine_response_content_type(accept_header):
    if accept_header and 'application/vnd.collection+json' in accept_header:
        return 'application/vnd.collection+json'
    else:
        return 'application/json'

# routes
# default
@app.route('/')
def callback():
    LOGGER.info('ROUTE: /')
    abort(404, "not found")

# billboard route - '^\/api$'
@app.route(path='/api/', method='GET')
def callback():
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/')
    response.set_header('Date', DATETIMESTAMP)

    try:
        # check accepts and content type
        # content_type = request.headers.get('Content-Type')
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))
        response.status = 200

        collection = dict()
        links = list()

        link = LinkCJ(ROOT + 'movies-alps.xml', 'profile', 'profile')
        links.append(link.to_dict())

        link = LinkCJ(ROOT + 'movies', 'Movies Collection', 'movies')
        links.append(link.to_dict())

        link = LinkCJ(ROOT + 'persons', 'Persons Collection', 'persons')
        links.append(link.to_dict())

        link = LinkCJ(ROOT + 'docs', 'API Documentation', 'documentation')
        links.append(link.to_dict())

        collection['version'] = '1.0'
        collection['href'] = ROOT
        collection['links'] = links

        response_body = {
            'collection': collection,
        }

        return json.dumps(response_body)
    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movies collection route - '^\/api\/movies$'
@app.route(path='/api/movies', method='GET')
def callback():
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies')
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        # connect to mongodb
        mongoengine.connect('api')

        movie_collection = MoviesCollectionTemplateCJ(ROOT)

        limit = 5
        for movie in Movie.objects[:limit]:
            movie_item = MovieItemCJ(ROOT, movie)
            movie_collection.items.append(movie_item.to_dict())

        # TODO add come count logic here
        link = LinkCJ(ROOT + 'movies', 'first', 'first')
        movie_collection.links.append(link.to_dict())

        link = LinkCJ(ROOT + 'movies?start=6', 'next', 'next')
        movie_collection.links.append(link.to_dict())

        # TODO paging next, prev, last

        response.status = 200
        response_body = movie_collection.to_json()
        return response_body

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies\/.*'
@app.route(path='/api/movies/<mid:re:.+>', method='GET')
def callback(mid):
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies/' + mid)
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        # connect to mongodb and lookup movie with matching sysid
        mongoengine.connect('api')
        movie = Movie.objects.get(sysid=mid)
        movie_item = MovieItemCJ(ROOT, movie)
        movie_collection = MoviesCollectionTemplateCJ(ROOT)
        movie_collection.items.append(movie_item.to_dict())

        response.status = 200
        response_body = movie_collection.to_json()
        return response_body

    except DoesNotExist:
        LOGGER.warning('Does not exist warning, cannot find movie with sysid: ' + mid)
        response.status = 204
        return

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies$'
@app.route(path='/api/movies', method='POST')
def callback():
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies')
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        if 'application/json' not in request.headers.get(
                'Content-Type') and 'application/vnd.collection+json' not in request.headers.get('Content-Type'):
            LOGGER.error('Unsupported media type sent')
            response.status = 415
            response_body = ErrorCJ(ROOT, 'Invalid Content-Type', 415,
                                    'application/json and application/vnd.collection+json supported')
            return response_body.to_json()

        request_json = json_loads(request._get_body_string())
        if not request_json:
            raise Exception

        # debug
        # print request_json
        # print len(request_json["template"]["data"])

        # connect to mongodb
        mongoengine.connect('api')

        # TODO Need to note that we are only expecting a single item to be
        # TODO created at any time
        movie_dict = dict()
        for item in request_json["template"]["data"]:
            movie_dict[item["name"]] = item["value"]

        # new movie
        movie = Movie()
        movie.sysid = str(uuid.uuid4()).replace("-", "")
        movie.created_on = datetime.datetime.now
        movie.updated_on = datetime.datetime.now
        movie.decode(movie_dict)
        movie.save()

        # send the response
        response.set_header('Location', ROOT + 'movies/' + movie.sysid)
        response.status = 201
        movie = None
        return

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies\/.*'
@app.route(path='/api/movies/<id:re:.+>', method='PUT')
def callback(mid):
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies/' + mid)
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))
        response.status = 200

        with open('movie_item.json') as json_data:
            response_body = json.load(json_data)
            json_data.close()
        return json.dumps(response_body)

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies\/.*'
@app.route(path='/api/movies/<mid:re:.+>', method='DELETE')
def callback(mid):
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies/' + mid)
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        # connect to mongodb and delete movie with matching sysid
        mongoengine.connect('api')

        # make sure we are only deleting one record
        movies = Movie.objects(sysid=mid)[:1]
        movies.delete()
        response.status = 204
        return

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# main
run(app, host=HOST, port=PORT, debug=True)
<commit_msg>Python PUT to Update Working<commit_after>"""
BottlePy API Implementation for ISWQ2014
"""
import logging

logging.basicConfig(format='%(asctime)s %(message)s')

LOGGER = logging.getLogger('app')
LOGGER.setLevel('INFO')

import json
from bottle import *
from models.Movie import *
from templates.LinkCJ import LinkCJ
from templates.ErrorCJ import ErrorCJ
from templates.MoviesTemplateCJ import MoviesTemplateCJ
from templates.MovieItemCJ import MovieItemCJ
from templates.MoviesCollectionTemplateCJ import MoviesCollectionTemplateCJ
import mongoengine
import datetime

HOST = 'localhost'
PORT = 1337
PATH = '/api/'
BASE = 'http://'
ROOT = BASE + HOST + ':' + str(PORT) + PATH
DATETIMESTAMP = datetime.datetime.utcnow().isoformat() + 'Z'

# application setting
app = default_app();
app.config['autojson'] = False

# setup response Content-Type
def determine_response_content_type(accept_header):
    if accept_header and 'application/vnd.collection+json' in accept_header:
        return 'application/vnd.collection+json'
    else:
        return 'application/json'

# routes
# default
@app.route('/')
def callback_default():
    LOGGER.info('ROUTE: /')
    abort(404, "not found")

# billboard route - '^\/api$'
@app.route(path='/api/', method='GET')
def callback_get_billboard():
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/')
    response.set_header('Date', DATETIMESTAMP)

    try:
        # check accepts and content type
        # content_type = request.headers.get('Content-Type')
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))
        response.status = 200

        collection = dict()
        links = list()

        link = LinkCJ(ROOT + 'movies-alps.xml', 'profile', 'profile')
        links.append(link.to_dict())

        link = LinkCJ(ROOT + 'movies', 'Movies Collection', 'movies')
        links.append(link.to_dict())

        link = LinkCJ(ROOT + 'persons', 'Persons Collection', 'persons')
        links.append(link.to_dict())

        link = LinkCJ(ROOT + 'docs', 'API Documentation', 'documentation')
        links.append(link.to_dict())

        collection['version'] = '1.0'
        collection['href'] = ROOT
        collection['links'] = links

        response_body = {
            'collection': collection,
        }

        return json.dumps(response_body)
    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movies collection route - '^\/api\/movies$'
@app.route(path='/api/movies', method='GET')
def callback_get_movies():
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies')
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        # connect to mongodb
        mongoengine.connect('api')

        movie_collection = MoviesCollectionTemplateCJ(ROOT)

        limit = 5
        for movie in Movie.objects[:limit]:
            movie_item = MovieItemCJ(ROOT, movie)
            movie_collection.items.append(movie_item.to_dict())

        # TODO add come count logic here
        link = LinkCJ(ROOT + 'movies', 'first', 'first')
        movie_collection.links.append(link.to_dict())

        link = LinkCJ(ROOT + 'movies?start=6', 'next', 'next')
        movie_collection.links.append(link.to_dict())

        # TODO paging next, prev, last

        response.status = 200
        response_body = movie_collection.to_json()
        return response_body

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies\/.*'
@app.route(path='/api/movies/<mid:re:.+>', method='GET')
def callback_get_movie(mid):
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies/' + mid)
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        # connect to mongodb and lookup movie with matching sysid
        mongoengine.connect('api')
        movie = Movie.objects.get(sysid=mid)
        movie_item = MovieItemCJ(ROOT, movie)
        movie_collection = MoviesCollectionTemplateCJ(ROOT)
        movie_collection.items.append(movie_item.to_dict())

        response.status = 200
        response_body = movie_collection.to_json()
        return response_body

    except DoesNotExist:
        LOGGER.warning('Does not exist warning, cannot find movie with sysid: ' + mid)
        response.status = 204
        return

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies$'
@app.route(path='/api/movies', method='POST')
def callback_post():
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies')
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        if 'application/json' not in request.headers.get(
                'Content-Type') and 'application/vnd.collection+json' not in request.headers.get('Content-Type'):
            LOGGER.error('Unsupported media type sent')
            response.status = 415
            response_body = ErrorCJ(ROOT, 'Invalid Content-Type', 415,
                                    'application/json and application/vnd.collection+json supported')
            return response_body.to_json()

        request_json = json_loads(request._get_body_string())
        if not request_json:
            raise Exception

        # debug
        # print request_json

        # connect to mongodb
        mongoengine.connect('api')

        # TODO Need to note that we are only expecting a single item to be
        # TODO created at any time
        movie_dict = dict()
        for item in request_json["template"]["data"]:
            movie_dict[item["name"]] = item["value"]

        # new movie
        movie = Movie()
        movie.sysid = str(uuid.uuid4()).replace("-", "")
        movie.created_on = datetime.datetime.now
        movie.updated_on = datetime.datetime.now
        movie.decode(movie_dict)
        movie.save()

        # send the response
        response.set_header('Location', ROOT + 'movies/' + movie.sysid)
        response.status = 201
        movie = None
        return

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies\/.*'
@app.route(path='/api/movies/<mid:re:.+>', method='PUT')
def callback_put(mid):
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies/' + mid)
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        if 'application/json' not in request.headers.get(
                'Content-Type') and 'application/vnd.collection+json' not in request.headers.get('Content-Type'):
            LOGGER.error('Unsupported media type sent')
            response.status = 415
            response_body = ErrorCJ(ROOT, 'Invalid Content-Type', 415,
                                    'application/json and application/vnd.collection+json supported')
            return response_body.to_json()

        request_json = json_loads(request._get_body_string())
        if not request_json:
            raise Exception

        # debug
        # print request_json

        # connect to mongodb
        mongoengine.connect('api')

        movie = Movie.objects.get(sysid=mid)

        # if we don't find a movie with this id
        if not movie:
            LOGGER.warn('movie.sysid' + mid + ' not found, unable to update')
            response.status = 204
            return

        # TODO Cleanup
        if "data" not in request_json["template"]:
            print 'error data not found'

        movie_dict = dict()
        for item in request_json["template"]["data"]:
            movie_dict[item["name"]] = item["value"]

        movie.updated_on = datetime.datetime.now
        movie.decode(movie_dict)
        movie.save()

        # testing
        movie_item = MovieItemCJ(ROOT, movie)
        movie_collection = MoviesCollectionTemplateCJ(ROOT)
        movie_collection.items.append(movie_item.to_dict())

        response.status = 200
        response_body = movie_collection.to_json()
        return response_body

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# movie item route - '^\/api\/movies\/.*'
@app.route(path='/api/movies/<mid:re:.+>', method='DELETE')
def callback_delete(mid):
    LOGGER.info('Logging Request: METHOD: ' + request.method + ' => ROUTE: /api/movies/' + mid)
    response.set_header('Date', DATETIMESTAMP)

    try:
        accept = request.headers.get('Accept')
        response.set_header('Content-Type', determine_response_content_type(accept))

        # connect to mongodb and delete movie with matching sysid
        mongoengine.connect('api')

        # make sure we are only deleting one record
        movies = Movie.objects(sysid=mid)[:1]
        movies.delete()
        response.status = 204
        return

    except Exception as e:
        LOGGER.error('Unexpected exception ' + str(e))
        response.status = 500
        response_body = ErrorCJ(ROOT, 'Error Title', 500, str(e))
        return response_body.to_json()

# main
run(app, host=HOST, port=PORT, debug=True)
<|endoftext|>


<commit_before>from __future__ import unicode_literals, division, absolute_import
import os
import re
import urlparse
from collections import defaultdict

import jsonschema

from flexget.utils import qualities, template

schema_paths = {}

def register_schema(path, schema):
    """
    Register `schema` to be available at `path` for $refs

    :param path: Path to make schema available
    :param schema: The schema, or function which returns the schema
    """
    schema_paths[path] = schema

def resolve_ref(uri):
    """
    Finds and returns a schema pointed to by `uri` that has been registered in the register_schema function.
    """
    parsed = urlparse.urlparse(uri)
    if parsed.path in schema_paths:
        schema = schema_paths[parsed.path]
        if callable(schema):
            return schema(**dict(urlparse.parse_qsl(parsed.query)))
        return schema
    raise jsonschema.RefResolutionError("%s could not be resolved" % uri)

class RefResolver(jsonschema.RefResolver):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('handlers', {'': resolve_ref})
        super(RefResolver, self).__init__(*args, **kwargs)

format_checker = jsonschema.FormatChecker(('regex', 'email'))
format_checker.checks('quality', raises=ValueError)(qualities.get)
format_checker.checks('quality_requirements', raises=ValueError)(qualities.Requirements)

@format_checker.checks('file')
def is_file(instance):
    return os.path.isfile(os.path.expanduser(instance))

@format_checker.checks('path')
def is_path(instance):
    # If string replacement is allowed, only validate the part of the
    # path before the first identifier to be replaced
    pat = re.compile(r'{[{%].*[}%]}')
    result = pat.search(instance)
    if result:
        instance = os.path.dirname(instance[0:result.start()])
    return os.path.isdir(os.path.expanduser(instance))

#TODO: jsonschema has a format checker for uri if rfc3987 is installed, perhaps we should use that
@format_checker.checks('url')
def is_url(instance):
    regexp = ('(' + '|'.join(['ftp', 'http', 'https', 'file', 'udp']) +
              '):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?')
    return re.match(regexp, instance)

@format_checker.checks('interval', raises=ValueError)
def is_interval(instance):
    regexp = r'^\d+ (second|minute|hour|day|week)s?$'
    if not re.match(regexp, instance):
        raise ValueError("should be in format 'x (seconds|minutes|hours|days|weeks)'")
    return True

class ValidationError(jsonschema.ValidationError):
    """
     Overrides error messages from jsonschema with custom ones for FlexGet

    """

    @property
    def error_with_path(self):
        return "[/%s] %s" % ('/'.join(map(unicode, self.path)), self.message)

    @property
    def message(self):
        custom_error = self.schema.get('error_%s' % self.validator, self.schema.get('error'))
        if custom_error:
            return template.render(custom_error, self.__dict__)
        elif hasattr(self, 'message_%s' % self.validator):
            return getattr(self, 'message_%s' % self.validator)()
        return self._message

    @message.setter
    def message(self, value):
        self._message = value

    def message_type(self):
        return self._message.replace("'object'", "'dict'")

    def message_format(self):
        if self.cause:
            return unicode(self.cause)
        return self._message

    def message_enum(self):
        return 'Must be one of the following: %s' % ', '.join(map(unicode, self.validator_value))

    def message_additionalProperties(self):
        if self.validator_value is False:
            extras = set(jsonschema._find_additional_properties(self.instance, self.schema))
            if len(extras) == 1:
                return 'The key `%s` is not valid here.' % extras.pop()
            else:
                return 'The keys %s are not valid here.' % ', '.join('`%s`' % e for e in extras)
        return self._message

class SchemaValidator(jsonschema.Draft4Validator):
    """
    Validator for our schemas.
    Sets up local ref resolving and our custom format checkers.
    """
    def __init__(self, schema):
        resolver = RefResolver.from_schema(schema)
        super(SchemaValidator, self).__init__(schema, resolver=resolver, format_checker=format_checker)

    def iter_errors(self, instance, _schema=None):
        for e in super(SchemaValidator, self).iter_errors(instance, _schema=_schema):
            yield ValidationError.create_from(e)

    def validate_anyOf(self, *args, **kwargs):
        for error in super(SchemaValidator, self).validate_anyOf(*args, **kwargs):
            # Split the suberrors up by which subschema they are from
            subschema_errors = defaultdict(list)
            for sube in error.context:
                subschema_errors[sube.schema_path[0]].append(sube)
            # Find the subschemas that did not have a 'type' error validating the instance at this path
            no_type_errors = dict(subschema_errors)
            valid_types = set()
            for i, errors in subschema_errors.iteritems():
                for e in errors:
                    if e.validator == 'type' and not e.path:
                        # Remove from the no_type_errors dict
                        no_type_errors.pop(i, None)
                        # Add the valid types to the list of all valid types
                        if self.is_type(e.validator_value, 'string'):
                            valid_types.add(e.validator_value)
                        else:
                            valid_types.update(e.validator_value)
            if not no_type_errors:
                # If all of the branches had a 'type' error, create our own virtual type error with all possible types
                for e in self.descend(error.instance, {'type': valid_types}):
                    yield e
            elif len(no_type_errors) == 1:
                # If one of the possible schemas did not have a 'type' error, assume that is the intended one and issue
                # all errors from that subschema
                for e in no_type_errors.values()[0]:
                    e.schema_path.extendleft(reversed(error.schema_path))
                    e.path.extendleft(reversed(error.path))
                    yield e
            else:
                yield error

def one_or_more(schema):
    """
    Helper function to construct a schema that validates items matching `schema` or an array
    containing items matching `schema`.

    """

    return {
        "anyOf": [
            schema,
            {"type": "array", "items": schema, "minItems": 1}
        ]
    }
<commit_msg>Clean up the type error message<commit_after>from __future__ import unicode_literals, division, absolute_import
import os
import re
import urlparse
from collections import defaultdict

import jsonschema

from flexget.utils import qualities, template

schema_paths = {}

def register_schema(path, schema):
    """
    Register `schema` to be available at `path` for $refs

    :param path: Path to make schema available
    :param schema: The schema, or function which returns the schema
    """
    schema_paths[path] = schema

def resolve_ref(uri):
    """
    Finds and returns a schema pointed to by `uri` that has been registered in the register_schema function.
    """
    parsed = urlparse.urlparse(uri)
    if parsed.path in schema_paths:
        schema = schema_paths[parsed.path]
        if callable(schema):
            return schema(**dict(urlparse.parse_qsl(parsed.query)))
        return schema
    raise jsonschema.RefResolutionError("%s could not be resolved" % uri)

class RefResolver(jsonschema.RefResolver):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('handlers', {'': resolve_ref})
        super(RefResolver, self).__init__(*args, **kwargs)

format_checker = jsonschema.FormatChecker(('regex', 'email'))
format_checker.checks('quality', raises=ValueError)(qualities.get)
format_checker.checks('quality_requirements', raises=ValueError)(qualities.Requirements)

@format_checker.checks('file')
def is_file(instance):
    return os.path.isfile(os.path.expanduser(instance))

@format_checker.checks('path')
def is_path(instance):
    # If string replacement is allowed, only validate the part of the
    # path before the first identifier to be replaced
    pat = re.compile(r'{[{%].*[}%]}')
    result = pat.search(instance)
    if result:
        instance = os.path.dirname(instance[0:result.start()])
    return os.path.isdir(os.path.expanduser(instance))

#TODO: jsonschema has a format checker for uri if rfc3987 is installed, perhaps we should use that
@format_checker.checks('url')
def is_url(instance):
    regexp = ('(' + '|'.join(['ftp', 'http', 'https', 'file', 'udp']) +
              '):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?')
    return re.match(regexp, instance)

@format_checker.checks('interval', raises=ValueError)
def is_interval(instance):
    regexp = r'^\d+ (second|minute|hour|day|week)s?$'
    if not re.match(regexp, instance):
        raise ValueError("should be in format 'x (seconds|minutes|hours|days|weeks)'")
    return True

class ValidationError(jsonschema.ValidationError):
    """
     Overrides error messages from jsonschema with custom ones for FlexGet

    """

    @property
    def error_with_path(self):
        return "[/%s] %s" % ('/'.join(map(unicode, self.path)), self.message)

    @property
    def message(self):
        custom_error = self.schema.get('error_%s' % self.validator, self.schema.get('error'))
        if custom_error:
            return template.render(custom_error, self.__dict__)
        elif hasattr(self, 'message_%s' % self.validator):
            return getattr(self, 'message_%s' % self.validator)()
        return self._message

    @message.setter
    def message(self, value):
        self._message = value

    def message_type(self):
        if isinstance(self.validator_value, basestring):
            valid_types = [self.validator_value]
        else:
            valid_types = list(self.validator_value)
        # Replace some types with more pythony ones
        replace = {'object': 'dict', 'array': 'list'}
        valid_types = [replace.get(t, t) for t in valid_types]
        # Make valid_types into an english list, with commas and 'or'
        valid_types = ', '.join(valid_types[:-2] + ['']) + ' or '.join(valid_types[-2:])
        if isinstance(self.instance, dict):
            return 'Got a dict, expected: %s' % valid_types
        if isinstance(self.instance, list):
            return 'Got a list, expected: %s' % valid_types
        return 'Got `%s`, expected: %s' % (self.instance, valid_types)

    def message_format(self):
        if self.cause:
            return unicode(self.cause)
        return self._message

    def message_enum(self):
        return 'Must be one of the following: %s' % ', '.join(map(unicode, self.validator_value))

    def message_additionalProperties(self):
        if self.validator_value is False:
            extras = set(jsonschema._find_additional_properties(self.instance, self.schema))
            if len(extras) == 1:
                return 'The key `%s` is not valid here.' % extras.pop()
            else:
                return 'The keys %s are not valid here.' % ', '.join('`%s`' % e for e in extras)
        return self._message

class SchemaValidator(jsonschema.Draft4Validator):
    """
    Validator for our schemas.
    Sets up local ref resolving and our custom format checkers.
    """
    def __init__(self, schema):
        resolver = RefResolver.from_schema(schema)
        super(SchemaValidator, self).__init__(schema, resolver=resolver, format_checker=format_checker)

    def iter_errors(self, instance, _schema=None):
        for e in super(SchemaValidator, self).iter_errors(instance, _schema=_schema):
            yield ValidationError.create_from(e)

    def validate_anyOf(self, *args, **kwargs):
        for error in super(SchemaValidator, self).validate_anyOf(*args, **kwargs):
            # Split the suberrors up by which subschema they are from
            subschema_errors = defaultdict(list)
            for sube in error.context:
                subschema_errors[sube.schema_path[0]].append(sube)
            # Find the subschemas that did not have a 'type' error validating the instance at this path
            no_type_errors = dict(subschema_errors)
            valid_types = set()
            for i, errors in subschema_errors.iteritems():
                for e in errors:
                    if e.validator == 'type' and not e.path:
                        # Remove from the no_type_errors dict
                        no_type_errors.pop(i, None)
                        # Add the valid types to the list of all valid types
                        if self.is_type(e.validator_value, 'string'):
                            valid_types.add(e.validator_value)
                        else:
                            valid_types.update(e.validator_value)
            if not no_type_errors:
                # If all of the branches had a 'type' error, create our own virtual type error with all possible types
                for e in self.descend(error.instance, {'type': valid_types}):
                    yield e
            elif len(no_type_errors) == 1:
                # If one of the possible schemas did not have a 'type' error, assume that is the intended one and issue
                # all errors from that subschema
                for e in no_type_errors.values()[0]:
                    e.schema_path.extendleft(reversed(error.schema_path))
                    e.path.extendleft(reversed(error.path))
                    yield e
            else:
                yield error

def one_or_more(schema):
    """
    Helper function to construct a schema that validates items matching `schema` or an array
    containing items matching `schema`.

    """

    return {
        "anyOf": [
            schema,
            {"type": "array", "items": schema, "minItems": 1}
        ]
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from datetime import datetime
from os.path import join, abspath, dirname, isfile
from fabric.api import env, puts, abort, cd, hide, task
from fabric.operations import sudo, settings, run
from fabric.contrib import console
from fabric.contrib.files import upload_template

from fabric.colors import _wrap_with, green

green_bg = _wrap_with('42')
red_bg = _wrap_with('41')
fagungis_path = dirname(abspath(__file__))

##########################
## START Fagungis tasks ##
##########################

@task
def setup():
    #  test configuration start
    if not test_configuration():
        if not console.confirm("Configuration test %s! Do you want to continue?" % red_bg('failed'), default=False):
            abort("Aborting at user request.")
    #  test configuration end
    if env.ask_confirmation:
        if not console.confirm("Are you sure you want to setup %s?" % red_bg(env.project.upper()), default=False):
            abort("Aborting at user request.")
    puts(green_bg('Start setup...'))
    start_time = datetime.now()

    _verify_sudo
    _install_dependencies()
    _create_django_user()
    _setup_directories()
    _hg_clone()
    _install_virtualenv()
    _create_virtualenv()
    _install_gunicorn()
    _install_requirements()
    _upload_nginx_conf()
    _upload_rungunicorn_script()
    _upload_supervisord_conf()

    end_time = datetime.now()
    finish_message = '[%s] Correctly finished in %i seconds' % \
    (green_bg(end_time.strftime('%H:%M:%S')), (end_time - start_time).seconds)
    puts(finish_message)

@task
def deploy():
    #  test configuration start
    if not test_configuration():
        if not console.confirm("Configuration test %s! Do you want to continue?" % red_bg('failed'), default=False):
            abort("Aborting at user request.")
    #  test configuration end
    _verify_sudo()
    if env.ask_confirmation:
        if not console.confirm("Are you sure you want to deploy in %s?" % red_bg(env.project.upper()), default=False):
            abort("Aborting at user request.")
    puts(green_bg('Start setup...'))
    start_time = datetime.now()

    hg_pull()
    _install_requirements()
    _upload_nginx_conf()
    _upload_rungunicorn_script()
    _prepare_django_project()
    _prepare_media_path()
    _supervisor_restart()

    end_time = datetime.now()
    finish_message = '[%s] Correctly deployed in %i seconds' % \
    (green_bg(end_time.strftime('%H:%M:%S')), (end_time - start_time).seconds)
    puts(finish_message)

@task
def hg_pull():
    with cd(env.code_root):
        run('hg pull -u')

@task
def test_configuration(verbose=True):
    errors = []
    parameters_info = []
    if 'project' not in env or not env.project:
        errors.append('Project name missing')
    elif verbose:
        parameters_info.append(('Project name', env.project))
    if 'repository' not in env or not env.repository:
        errors.append('Repository url missing')
    elif verbose:
        parameters_info.append(('Repository url', env.repository))
    if 'hosts' not in env or not env.hosts:
        errors.append('Hosts configuration missing')
    elif verbose:
        parameters_info.append(('Hosts', env.hosts))
    if 'django_user' not in env or not env.django_user:
        errors.append('Django user missing')
    elif verbose:
        parameters_info.append(('Django user', env.django_user))
    if 'django_user_group' not in env or not env.django_user_group:
        errors.append('Django user group missing')
    elif verbose:
        parameters_info.append(('Django user group', env.django_user_group))
    if 'django_user_home' not in env or not env.django_user_home:
        errors.append('Django user home dir missing')
    elif verbose:
        parameters_info.append(('Django user home dir', env.django_user_home))
    if 'projects_path' not in env or not env.projects_path:
        errors.append('Projects path configuration missing')
    elif verbose:
        parameters_info.append(('Projects path', env.projects_path))
    if 'code_root' not in env or not env.code_root:
        errors.append('Code root configuration missing')
    elif verbose:
        parameters_info.append(('Code root', env.code_root))
    if 'django_project_root' not in env or not env.django_project_root:
        errors.append('Django project root configuration missing')
    elif verbose:
        parameters_info.append(('Django project root', env.django_project_root))
    if 'django_media_path' not in env or not env.django_media_path:
        errors.append('Django media path configuration missing')
    elif verbose:
        parameters_info.append(('Django media path', env.django_media_path))
    if 'django_static_path' not in env or not env.django_static_path:
        errors.append('Django static path configuration missing')
    elif verbose:
        parameters_info.append(('Django static path', env.django_static_path))
    if 'south_used' not in env:
        errors.append('"south_used" configuration missing')
    elif verbose:
        parameters_info.append(('south_used', env.south_used))
    if 'virtenv' not in env or not env.virtenv:
        errors.append('virtenv configuration missing')
    elif verbose:
        parameters_info.append(('virtenv', env.virtenv))
    if 'virtenv_options' not in env or not env.virtenv_options:
        errors.append('"virtenv_options" configuration missing, you must have at least one option')
    elif verbose:
        parameters_info.append(('virtenv_options', env.virtenv_options))
    if 'ask_confirmation' not in env:
        errors.append('"ask_confirmation" configuration missing')
    elif verbose:
        parameters_info.append(('ask_confirmation', env.ask_confirmation))
    if 'gunicorn_bind' not in env or not env.gunicorn_bind:
        errors.append('"gunicorn_bind" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_bind', env.gunicorn_bind))
    if 'gunicorn_logfile' not in env or not env.gunicorn_logfile:
        errors.append('"gunicorn_logfile" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_logfile', env.gunicorn_logfile))
    if 'gunicorn_workers' not in env or not env.gunicorn_workers:
        errors.append('"gunicorn_workers" configuration missing, you must have at least one worker')
    elif verbose:
        parameters_info.append(('gunicorn_workers', env.gunicorn_workers))
    if 'gunicorn_worker_class' not in env or not env.gunicorn_worker_class:
        errors.append('"gunicorn_worker_class" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_worker_class', env.gunicorn_worker_class))
    if 'gunicorn_loglevel' not in env or not env.gunicorn_loglevel:
        errors.append('"gunicorn_loglevel" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_loglevel', env.gunicorn_loglevel))
    if 'nginx_server_name' not in env or not env.nginx_server_name:
        errors.append('"nginx_server_name" configuration missing')
    elif verbose:
        parameters_info.append(('nginx_server_name', env.nginx_server_name))
    if 'supervisorctl' not in env or not env.supervisorctl:
        errors.append('"supervisorctl" configuration missing')
    elif verbose:
        parameters_info.append(('supervisorctl', env.supervisorctl))
    if 'supervisor_autostart' not in env or not env.supervisor_autostart:
        errors.append('"supervisor_autostart" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_autostart', env.supervisor_autostart))
    if 'supervisor_autorestart' not in env or not env.supervisor_autorestart:
        errors.append('"supervisor_autorestart" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_autorestart', env.supervisor_autorestart))
    if 'supervisor_redirect_stderr' not in env or not env.supervisor_redirect_stderr:
        errors.append('"supervisor_redirect_stderr" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_redirect_stderr', env.supervisor_redirect_stderr))
    if 'supervisor_stdout_logfile' not in env or not env.supervisor_stdout_logfile:
        errors.append('"supervisor_stdout_logfile" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_stdout_logfile', env.supervisor_stdout_logfile))

    if errors:
        if len(errors) == 26:
            ''' all configuration missing '''
            puts('Configuration missing! Please read README.rst first or go ahead at your own risk.')
        else:
            puts('Configuration test revealed %i errors:' % len(errors))
            puts('%s\n\n* %s\n' % ('-' * 37, '\n* '.join(errors)))
            puts('-' * 40)
            puts('Please fix them or go ahead at your own risk.')
        return False
    elif verbose:
        for parameter in parameters_info:
            parameter_formatting = "'%s'" if isinstance(parameter[1], str) else "%s"
            parameter_value = parameter_formatting % parameter[1]
            puts('%s %s' % (parameter[0].ljust(27), green(parameter_value)))
    puts('Configuration tests passed!')
    return True

########################
## END Fagungis tasks ##
########################

def _create_django_user():
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = sudo('useradd -d %(django_user_home)s -m -r %(django_user)s' % env)
        if 'already exists' in res:
            puts('User \'%(django_user)s\' already exists, will not be changed.' % env)
            return
        #  set password
        sudo('passwd %(django_user)s' % env)

def _verify_sudo():
    ''' we just check if the user is sudoers '''
    sudo('cd .')

def _install_nginx():
    # add nginx stable ppa
    sudo('add-apt-repository ppa:nginx/stable')
    sudo('apt-get update')
    sudo('apt-get install nginx')
    sudo('/etc/init.d/nginx start')

def _install_dependencies():
    ''' Ensure those Debian/Ubuntu packages are installed '''
    packages = [
        'mercurial',
        'python-pip',
        'supervisor',
    ]
    sudo('apt-get install %s' % ' '.join(packages))
    _install_nginx()
    sudo('pip install --upgrade pip')

def _install_requirements():
    ''' you must have a file called requirements.txt in your project root'''
    requirements = join(env.code_root, 'requirements.txt')
    virtenvrun('pip install -r %s' % requirements)

def _install_gunicorn():
    """ force gunicorn installation into your virtualenv, even if it's installed globally.
    for more details: https://github.com/benoitc/gunicorn/pull/280 """
    virtenvrun('pip install -I gunicorn')

def _install_virtualenv():
    sudo('pip install virtualenv')

def _create_virtualenv():
    sudo('virtualenv --%s %s' % (' --'.join(env.virtenv_options), env.virtenv))

def _setup_directories():
    sudo('mkdir -p %(projects_path)s' % env)
    sudo('mkdir -p %(django_user_home)s/logs/nginx' % env)
    sudo('mkdir -p %(django_user_home)s/logs/projects' % env)
    sudo('mkdir -p %(django_user_home)s/configs/nginx' % env)
    sudo('mkdir -p %(django_user_home)s/configs/supervisord' % env)
    sudo('mkdir -p %(django_user_home)s/scripts' % env)
    sudo('mkdir -p %(django_user_home)s/htdocs' % env)
    sudo('mkdir -p %(django_user_home)s/tmp' % env)
    sudo('mkdir -p %(virtenv)s' % env)
    sudo('echo "nothing here " > %(django_user_home)s/htdocs/index.html' % env)

def virtenvrun(command):
    activate = 'source %s/bin/activate' % env.virtenv
    run(activate + ' && ' + command)

def _hg_clone():
    run('hg clone %s %s' % (env.repository, env.code_root))

def _test_nginx_conf():
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = sudo('nginx -t -c /etc/nginx/nginx.conf')
    if 'test failed' in res:
        abort(red_bg('NGINX configuration test failed! Please review your parameters.'))

def _upload_nginx_conf():
    ''' upload nginx conf '''
    nginx_file = '%(django_user_home)s/configs/nginx/%(project)s.conf' % env
    if isfile('conf/nginx.conf'):
        ''' we use user defined nginx.conf template '''
        template = 'conf/nginx.conf'
    else:
        template = '%s/conf/nginx.conf' % fagungis_path
    upload_template(template, nginx_file,
                    context=env, backup=False)
    sudo('ln -sf %s /etc/nginx/sites-enabled/' % nginx_file)
    _test_nginx_conf()
    sudo('nginx -s reload')

def _reload_supervisorctl():
    sudo('%(supervisorctl)s reread' % env)
    sudo('%(supervisorctl)s reload' % env)

def _upload_supervisord_conf():
    ''' upload supervisor conf '''
    supervisord_conf_file = '%(django_user_home)s/configs/supervisord/%(project)s.conf' % env
    if isfile('conf/supervisord.conf'):
        ''' we use user defined supervisord.conf template '''
        template = 'conf/supervisord.conf'
    else:
        template = '%s/conf/supervisord.conf' % fagungis_path
    upload_template(template, supervisord_conf_file,
                    context=env, backup=False)
    sudo('ln -sf %s /etc/supervisor/conf.d/' % supervisord_conf_file)
    _reload_supervisorctl()

def _prepare_django_project():
    with cd(env.django_project_root):
        virtenvrun('./manage.py syncdb --noinput --verbosity=1')
        if env.south_used:
            virtenvrun('./manage.py migrate --noinput --verbosity=1')
        virtenvrun('./manage.py collectstatic --noinput')

def _prepare_media_path():
    sudo('chmod -R 775 %s' % env.django_media_path)

def _upload_rungunicorn_script():
    ''' upload rungunicorn conf '''
    script_file = '%(django_user_home)s/scripts/rungunicorn_%(project)s.sh' % env
    if isfile('scripts/rungunicorn.sh'):
        ''' we use user defined rungunicorn file '''
        template = 'scripts/rungunicorn.sh'
    else:
        template = '%s/scripts/rungunicorn.sh' % fagungis_path
    upload_template(template, script_file,
                    context=env, backup=False)
    sudo('chmod +x %s' % script_file)

def _supervisor_restart():
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = sudo('%(supervisorctl)s restart %(project)s' % env)
    if 'ERROR' in res:
        print red_bg("%s NOT STARTED!" % env.project)
    else:
        print green_bg("%s correctly started!" % env.project)
<commit_msg>bugfix in _create_django_user task<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from datetime import datetime
from os.path import join, abspath, dirname, isfile
from fabric.api import env, puts, abort, cd, hide, task
from fabric.operations import sudo, settings, run
from fabric.contrib import console
from fabric.contrib.files import upload_template

from fabric.colors import _wrap_with, green

green_bg = _wrap_with('42')
red_bg = _wrap_with('41')
fagungis_path = dirname(abspath(__file__))

##########################
## START Fagungis tasks ##
##########################

@task
def setup():
    #  test configuration start
    if not test_configuration():
        if not console.confirm("Configuration test %s! Do you want to continue?" % red_bg('failed'), default=False):
            abort("Aborting at user request.")
    #  test configuration end
    if env.ask_confirmation:
        if not console.confirm("Are you sure you want to setup %s?" % red_bg(env.project.upper()), default=False):
            abort("Aborting at user request.")
    puts(green_bg('Start setup...'))
    start_time = datetime.now()

    _verify_sudo
    _install_dependencies()
    _create_django_user()
    _setup_directories()
    _hg_clone()
    _install_virtualenv()
    _create_virtualenv()
    _install_gunicorn()
    _install_requirements()
    _upload_nginx_conf()
    _upload_rungunicorn_script()
    _upload_supervisord_conf()

    end_time = datetime.now()
    finish_message = '[%s] Correctly finished in %i seconds' % \
    (green_bg(end_time.strftime('%H:%M:%S')), (end_time - start_time).seconds)
    puts(finish_message)

@task
def deploy():
    #  test configuration start
    if not test_configuration():
        if not console.confirm("Configuration test %s! Do you want to continue?" % red_bg('failed'), default=False):
            abort("Aborting at user request.")
    #  test configuration end
    _verify_sudo()
    if env.ask_confirmation:
        if not console.confirm("Are you sure you want to deploy in %s?" % red_bg(env.project.upper()), default=False):
            abort("Aborting at user request.")
    puts(green_bg('Start setup...'))
    start_time = datetime.now()

    hg_pull()
    _install_requirements()
    _upload_nginx_conf()
    _upload_rungunicorn_script()
    _prepare_django_project()
    _prepare_media_path()
    _supervisor_restart()

    end_time = datetime.now()
    finish_message = '[%s] Correctly deployed in %i seconds' % \
    (green_bg(end_time.strftime('%H:%M:%S')), (end_time - start_time).seconds)
    puts(finish_message)

@task
def hg_pull():
    with cd(env.code_root):
        run('hg pull -u')

@task
def test_configuration(verbose=True):
    errors = []
    parameters_info = []
    if 'project' not in env or not env.project:
        errors.append('Project name missing')
    elif verbose:
        parameters_info.append(('Project name', env.project))
    if 'repository' not in env or not env.repository:
        errors.append('Repository url missing')
    elif verbose:
        parameters_info.append(('Repository url', env.repository))
    if 'hosts' not in env or not env.hosts:
        errors.append('Hosts configuration missing')
    elif verbose:
        parameters_info.append(('Hosts', env.hosts))
    if 'django_user' not in env or not env.django_user:
        errors.append('Django user missing')
    elif verbose:
        parameters_info.append(('Django user', env.django_user))
    if 'django_user_group' not in env or not env.django_user_group:
        errors.append('Django user group missing')
    elif verbose:
        parameters_info.append(('Django user group', env.django_user_group))
    if 'django_user_home' not in env or not env.django_user_home:
        errors.append('Django user home dir missing')
    elif verbose:
        parameters_info.append(('Django user home dir', env.django_user_home))
    if 'projects_path' not in env or not env.projects_path:
        errors.append('Projects path configuration missing')
    elif verbose:
        parameters_info.append(('Projects path', env.projects_path))
    if 'code_root' not in env or not env.code_root:
        errors.append('Code root configuration missing')
    elif verbose:
        parameters_info.append(('Code root', env.code_root))
    if 'django_project_root' not in env or not env.django_project_root:
        errors.append('Django project root configuration missing')
    elif verbose:
        parameters_info.append(('Django project root', env.django_project_root))
    if 'django_media_path' not in env or not env.django_media_path:
        errors.append('Django media path configuration missing')
    elif verbose:
        parameters_info.append(('Django media path', env.django_media_path))
    if 'django_static_path' not in env or not env.django_static_path:
        errors.append('Django static path configuration missing')
    elif verbose:
        parameters_info.append(('Django static path', env.django_static_path))
    if 'south_used' not in env:
        errors.append('"south_used" configuration missing')
    elif verbose:
        parameters_info.append(('south_used', env.south_used))
    if 'virtenv' not in env or not env.virtenv:
        errors.append('virtenv configuration missing')
    elif verbose:
        parameters_info.append(('virtenv', env.virtenv))
    if 'virtenv_options' not in env or not env.virtenv_options:
        errors.append('"virtenv_options" configuration missing, you must have at least one option')
    elif verbose:
        parameters_info.append(('virtenv_options', env.virtenv_options))
    if 'ask_confirmation' not in env:
        errors.append('"ask_confirmation" configuration missing')
    elif verbose:
        parameters_info.append(('ask_confirmation', env.ask_confirmation))
    if 'gunicorn_bind' not in env or not env.gunicorn_bind:
        errors.append('"gunicorn_bind" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_bind', env.gunicorn_bind))
    if 'gunicorn_logfile' not in env or not env.gunicorn_logfile:
        errors.append('"gunicorn_logfile" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_logfile', env.gunicorn_logfile))
    if 'gunicorn_workers' not in env or not env.gunicorn_workers:
        errors.append('"gunicorn_workers" configuration missing, you must have at least one worker')
    elif verbose:
        parameters_info.append(('gunicorn_workers', env.gunicorn_workers))
    if 'gunicorn_worker_class' not in env or not env.gunicorn_worker_class:
        errors.append('"gunicorn_worker_class" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_worker_class', env.gunicorn_worker_class))
    if 'gunicorn_loglevel' not in env or not env.gunicorn_loglevel:
        errors.append('"gunicorn_loglevel" configuration missing')
    elif verbose:
        parameters_info.append(('gunicorn_loglevel', env.gunicorn_loglevel))
    if 'nginx_server_name' not in env or not env.nginx_server_name:
        errors.append('"nginx_server_name" configuration missing')
    elif verbose:
        parameters_info.append(('nginx_server_name', env.nginx_server_name))
    if 'supervisorctl' not in env or not env.supervisorctl:
        errors.append('"supervisorctl" configuration missing')
    elif verbose:
        parameters_info.append(('supervisorctl', env.supervisorctl))
    if 'supervisor_autostart' not in env or not env.supervisor_autostart:
        errors.append('"supervisor_autostart" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_autostart', env.supervisor_autostart))
    if 'supervisor_autorestart' not in env or not env.supervisor_autorestart:
        errors.append('"supervisor_autorestart" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_autorestart', env.supervisor_autorestart))
    if 'supervisor_redirect_stderr' not in env or not env.supervisor_redirect_stderr:
        errors.append('"supervisor_redirect_stderr" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_redirect_stderr', env.supervisor_redirect_stderr))
    if 'supervisor_stdout_logfile' not in env or not env.supervisor_stdout_logfile:
        errors.append('"supervisor_stdout_logfile" configuration missing')
    elif verbose:
        parameters_info.append(('supervisor_stdout_logfile', env.supervisor_stdout_logfile))

    if errors:
        if len(errors) == 26:
            ''' all configuration missing '''
            puts('Configuration missing! Please read README.rst first or go ahead at your own risk.')
        else:
            puts('Configuration test revealed %i errors:' % len(errors))
            puts('%s\n\n* %s\n' % ('-' * 37, '\n* '.join(errors)))
            puts('-' * 40)
            puts('Please fix them or go ahead at your own risk.')
        return False
    elif verbose:
        for parameter in parameters_info:
            parameter_formatting = "'%s'" if isinstance(parameter[1], str) else "%s"
            parameter_value = parameter_formatting % parameter[1]
            puts('%s %s' % (parameter[0].ljust(27), green(parameter_value)))
    puts('Configuration tests passed!')
    return True

########################
## END Fagungis tasks ##
########################

def _create_django_user():
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = sudo('useradd -d %(django_user_home)s -m -r %(django_user)s' % env)
    if 'already exists' in res:
        puts('User \'%(django_user)s\' already exists, will not be changed.' % env)
        return
    #  set password
    sudo('passwd %(django_user)s' % env)

def _verify_sudo():
    ''' we just check if the user is sudoers '''
    sudo('cd .')

def _install_nginx():
    # add nginx stable ppa
    sudo('add-apt-repository ppa:nginx/stable')
    sudo('apt-get update')
    sudo('apt-get install nginx')
    sudo('/etc/init.d/nginx start')

def _install_dependencies():
    ''' Ensure those Debian/Ubuntu packages are installed '''
    packages = [
        'mercurial',
        'python-pip',
        'supervisor',
    ]
    sudo('apt-get install %s' % ' '.join(packages))
    _install_nginx()
    sudo('pip install --upgrade pip')

def _install_requirements():
    ''' you must have a file called requirements.txt in your project root'''
    requirements = join(env.code_root, 'requirements.txt')
    virtenvrun('pip install -r %s' % requirements)

def _install_gunicorn():
    """ force gunicorn installation into your virtualenv, even if it's installed globally.
    for more details: https://github.com/benoitc/gunicorn/pull/280 """
    virtenvrun('pip install -I gunicorn')

def _install_virtualenv():
    sudo('pip install virtualenv')

def _create_virtualenv():
    sudo('virtualenv --%s %s' % (' --'.join(env.virtenv_options), env.virtenv))

def _setup_directories():
    sudo('mkdir -p %(projects_path)s' % env)
    sudo('mkdir -p %(django_user_home)s/logs/nginx' % env)
    sudo('mkdir -p %(django_user_home)s/logs/projects' % env)
    sudo('mkdir -p %(django_user_home)s/configs/nginx' % env)
    sudo('mkdir -p %(django_user_home)s/configs/supervisord' % env)
    sudo('mkdir -p %(django_user_home)s/scripts' % env)
    sudo('mkdir -p %(django_user_home)s/htdocs' % env)
    sudo('mkdir -p %(django_user_home)s/tmp' % env)
    sudo('mkdir -p %(virtenv)s' % env)
    sudo('echo "nothing here " > %(django_user_home)s/htdocs/index.html' % env)

def virtenvrun(command):
    activate = 'source %s/bin/activate' % env.virtenv
    run(activate + ' && ' + command)

def _hg_clone():
    run('hg clone %s %s' % (env.repository, env.code_root))

def _test_nginx_conf():
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = sudo('nginx -t -c /etc/nginx/nginx.conf')
    if 'test failed' in res:
        abort(red_bg('NGINX configuration test failed! Please review your parameters.'))

def _upload_nginx_conf():
    ''' upload nginx conf '''
    nginx_file = '%(django_user_home)s/configs/nginx/%(project)s.conf' % env
    if isfile('conf/nginx.conf'):
        ''' we use user defined nginx.conf template '''
        template = 'conf/nginx.conf'
    else:
        template = '%s/conf/nginx.conf' % fagungis_path
    upload_template(template, nginx_file,
                    context=env, backup=False)
    sudo('ln -sf %s /etc/nginx/sites-enabled/' % nginx_file)
    _test_nginx_conf()
    sudo('nginx -s reload')

def _reload_supervisorctl():
    sudo('%(supervisorctl)s reread' % env)
    sudo('%(supervisorctl)s reload' % env)

def _upload_supervisord_conf():
    ''' upload supervisor conf '''
    supervisord_conf_file = '%(django_user_home)s/configs/supervisord/%(project)s.conf' % env
    if isfile('conf/supervisord.conf'):
        ''' we use user defined supervisord.conf template '''
        template = 'conf/supervisord.conf'
    else:
        template = '%s/conf/supervisord.conf' % fagungis_path
    upload_template(template, supervisord_conf_file,
                    context=env, backup=False)
    sudo('ln -sf %s /etc/supervisor/conf.d/' % supervisord_conf_file)
    _reload_supervisorctl()

def _prepare_django_project():
    with cd(env.django_project_root):
        virtenvrun('./manage.py syncdb --noinput --verbosity=1')
        if env.south_used:
            virtenvrun('./manage.py migrate --noinput --verbosity=1')
        virtenvrun('./manage.py collectstatic --noinput')

def _prepare_media_path():
    sudo('chmod -R 775 %s' % env.django_media_path)

def _upload_rungunicorn_script():
    ''' upload rungunicorn conf '''
    script_file = '%(django_user_home)s/scripts/rungunicorn_%(project)s.sh' % env
    if isfile('scripts/rungunicorn.sh'):
        ''' we use user defined rungunicorn file '''
        template = 'scripts/rungunicorn.sh'
    else:
        template = '%s/scripts/rungunicorn.sh' % fagungis_path
    upload_template(template, script_file,
                    context=env, backup=False)
    sudo('chmod +x %s' % script_file)

def _supervisor_restart():
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = sudo('%(supervisorctl)s restart %(project)s' % env)
    if 'ERROR' in res:
        print red_bg("%s NOT STARTED!" % env.project)
    else:
        print green_bg("%s correctly started!" % env.project)
<|endoftext|>