<commit_before>#-*- coding:utf8 -*-
# @author Thierry Stiegler thierry.stiegler@gmail.com

from fabric.api import *
from fabric.colors import red,yellow,green
from fabric.contrib.console import confirm
from fabric.utils import abort,error

from pprint import pprint

import os

# SETTINGS

IMAGES = ["base","postgresql", "mysql", "http", "php", "wsgi"]

NODES = [

## PROD CONTAINERS
    { "id": 1, "name" : "prodpostgresql01"     , "image":"qqch/postgresql:latest", "extra_port":"5432%(id)s:5432"  },
    { "id": 2, "name" : "prodmysql01"          , "image":"qqch/mysql:latest"     , "extra_port":"3306%(id)s:3306"  },
    { "id": 3, "name" : "prodjibaku01"         , "image":"qqch/wsgi:latest"      , "extra_port":"800%(id)s:80"     },
    { "id": 4, "name" : "prodjeanglode01"      , "image":"qqch/wsgi:latest"      , "extra_port":"800%(id)s:80"     },
    { "id": 5, "name" : "prodjeanglode02"      , "image":"qqch/php:latest"       , "extra_port":"800%(id)s:80"     },
    { "id": 6, "name" : "prodwp01"             , "image":"qqch/php:latest"       , "extra_port":"800%(id)s:80"     },
    { "id": 7, "name" : "prodtorzka01"         , "image":"qqch/php:latest"       , "extra_port":"800%(id)s:80"     },
    { "id": 8, "name" : "prodnolann01"         , "image":"qqch/http:latest"       , "extra_port":"800%(id)s:80"     },

## TEST CONTAINERS
    { "id": 9, "name" : "jouet01"         , "image":"qqch/base:latest"       ,  },
]

NETWORKS = [
    { "id": 1, "nodes": ("prodmysql01", "prodjeanglode02", "prodjeanglode01") },
    { "id": 2, "nodes": ("prodpostgresql01", "prodjibaku01")},
    { "id": 3, "nodes": ("prodmysql01", "prodwp01")},
    { "id": 4, "nodes": ("prodmysql01", "prodtorzka01")},

]

DOCKERFILES_ROOT = "/home/docker/dockerfiles"
CONTAINERS_DATAROOT = "/home/docker/volumes/"

DKCLEAN_CMD = "/home/docker/bin/dkclean.sh"
PIPEWORK_CMD = "sudo /home/docker/bin/pipework"
# TASKS

@task
def node(nodename,cmd=None):
    """
    The node mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab node:{nodename},boot : do a 'docker.io run' on each container, run it once time
      * fab node:{nodename},stop : stop all containers in the node
      * fab node:{nodename},start : do a 'docker.io start' on each container
      * fab node:{nodename},destroy : destroy all the node and clean docker's data (see dkclean)
    """

    generic_msg  = "\nAvailable commands are :\n  * start\n  * stop\n  * destroy\n  * boot"

    if cmd == "start":
        node_start(nodename)
        puts(green("node start [OK]"))
    elif cmd == "stop":
        node_stop(nodename)
        puts(green("node stop [OK]"))
    elif cmd == "boot":
        node_boot(nodename)
        puts(green("node boot [OK]"))
    elif cmd == "destroy":
        if confirm("Are you sure to destroy ? Everything you need is backuped ?",default=False):
            node_destroy(nodename)
            puts(green("node destroy [OK]"))
        else:
            abort(yellow("node destroy [CANCELED]"))
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

@task
def cluster(cmd=None):
    """
    The cluster mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab cluster:boot : do a 'docker.io run' on each container, run it once time
      * fab cluster:stop : stop all containers in the cluster
      * fab cluster:start : do a 'docker.io start' on each container
      * fab cluster:destroy : destroy all the cluster and clean docker's data (see dkclean)
    """

    generic_msg  = "\nAvailable commands are :\n  * start\n  * stop\n  * destroy\n  * boot"

    if cmd == "start":
        cluster_start()
        puts(green("cluster start [OK]"))
    elif cmd == "stop":
        cluster_stop()
        puts(green("cluster stop [OK]"))
    elif cmd == "boot":
        cluster_boot()
        puts(green("cluster boot [OK]"))
    elif cmd == "destroy":
        if confirm("Are you sure to destroy ? Everything you need is backuped ?",default=False):
            cluster_destroy()
            puts(green("cluster destroy [OK]"))
        else:
            abort(yellow("cluster destroy [CANCELED]"))
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

@task
def dockerui(cmd=None):
    """
    The dockerui mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab dockerui:boot : do a 'docker.io run' on each container, run it once time
      * fab dockerui:stop : stop all containers in the dockerui
      * fab dockerui:start : do a 'docker.io start' on each container
      * fab dockerui:destroy : destroy all the dockerui and clean docker's data (see dkclean)
    """

    generic_msg  = "\nAvailable commands are :\n  * start\n  * stop\n  * destroy\n  * boot"

    if cmd == "start":
        dockerui_start()
        puts(green("dockerui start [OK]"))
    elif cmd == "stop":
        dockerui_stop()
        puts(green("dockerui stop [OK]"))
    elif cmd == "boot":
        dockerui_boot()
        puts(green("dockerui boot [OK]"))
    elif cmd == "destroy":
        if confirm("Are you sure to destroy ? Everything you need is backuped ?",default=False):
            dockerui_destroy()
            puts(green("dockerui destroy [OK]"))
        else:
            abort(yellow("dockerui destroy [CANCELED]"))
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

@task
def build(*images):
    """
    Build given images in args. if args is empty it builds %s
    """ % IMAGES

    if len(images) <= 0:
        images = IMAGES

    for image in images:
        dirname = "qqch-%s" % (image,)
        options = {
          "image":image,
          "dpath" : os.path.join(DOCKERFILES_ROOT, dirname),
         }
        local("docker.io build -t qqch/%(image)s %(dpath)s" % options)
    dkclean()

@task
def dkclean():
    """
    Warn the user to run the dkclean
    """
    warn(yellow("Please run : %s" % DKCLEAN_CMD))

def dockerui_boot():
    local("docker.io run -h dockerui --name=dockerui -d -p 9000:9000 -v /var/run/docker.sock:/docker.sock crosbymichael/dockerui -e /docker.sock")

def dockerui_start():
    local("docker.io start dockerui")

def dockerui_stop():
    local("docker.io stop dockerui")

def dockerui_destroy():
    local("docker.io kill dockerui")
    dkclean()

def getports(defnode):
    options = {
      "convention" : "--publish=220%(id)s:22 --publish=490%(id)s:4949" ,
      "extra"      : "",
    }

    if defnode.has_key("extra_port"):
        options["extra"] = "--publish=%(extra_port)s" % defnode

    tpl = "%(convention)s %(extra)s" % options
    return tpl % defnode

def getvpath(defnode):
    return os.path.join(CONTAINERS_DATAROOT, defnode.get("name"))

def getlinks(defnode):
    r = ""
    for link in defnode.get("links", [] ):
        fragment = "--link=%s" % link
        r = r + fragment
    return r

CMD_BOOT = "docker.io run --hostname=%(name)s --name=%(name)s -v %(vpath)s:/data %(ports)s %(linking)s -d -t %(image)s  /sbin/my_init"
def cluster_boot():
    for defnode in NODES:
        defnode["ports"] = getports(defnode)
        defnode["vpath"] = getvpath(defnode)
        defnode["linking"] = getlinks(defnode)
        local("mkdir -p %(vpath)s" % defnode)
        cmd = CMD_BOOT % defnode
        local(cmd)
    network_up()

def node_find(nodename):
     """
     Search a node by it's name, return None if not found
     """
     for node in NODES:
          if node.get("name") == nodename:
              found = True
              return node

     puts(red("%s not found in cluster definition" % nodename))
     return None

def node_boot(nodename):
    defnode = node_find(nodename)
    if defnode:
        defnode["ports"] = getports(defnode)
        defnode["vpath"] = getvpath(defnode)
        defnode["linking"] = getlinks(defnode)
        local(CMD_BOOT % defnode)
        network_up(nodename)

def cluster_exec(cmd):
    for defnode in NODES :
        local(cmd % defnode)

def node_exec(nodename,cmd):
    defnode = node_find(nodename)
    if defnode:
        local(cmd % defnode)

CMD_STOP = "docker.io stop %(name)s"
def cluster_stop():
    cluster_exec(CMD_STOP)

def node_stop(nodename):
    node_exec(nodename, CMD_STOP)

CMD_DESTROY = "docker.io kill %(name)s"
def cluster_destroy():
    cluster_exec(CMD_DESTROY)
    dkclean()

def node_destroy(nodename):
    node_exec(nodename, CMD_DESTROY)
    dkclean()

CMD_START = "docker.io start %(name)s"
def cluster_start():
    cluster_exec(CMD_START)
    network_up()

def node_start(nodename):
    node_exec(nodename, CMD_START)
    network_up(nodename)

def conf_render(tpl):
    for defnode in NODES:
        puts(tpl % defnode)

@task
def conf(cmd=None):
    """
    The conf mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab conf:munin : generate the node tree for /etc/munin/munin.conf
      * fab conf:upstreams : generate the upstreams definition for nginx
      * fab conf:ufw : generate the ufw rules for making ssh available from the outside for every container
    """

    generic_msg  = "\nAvailable commands are :\n  * munin\n  * ssh"

    if cmd == "ssh":
        conf_munin()
    elif cmd == "upstreams":
        conf_upstreams()
    elif cmd == "ufw":
        puts(generic_msg)
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

def conf_munin():
    tpl = """
[docker;%(name)s]
    address 172.17.42.1
    port 490%(id)s
    use_node_name yes
"""
    conf_render(tpl)

def conf_upstreams():
    tpl ="""
upstream %(name)s {
    server 172.17.42.1:800%(id)s;
}
"""
    conf_render(tpl)

def conf_ufw():
    tpl = """
ufw allow out 220%(id)s
"""
    conf_render(tpl)

@task
def network_up(onlyfornode=None):
    """
    Build privates networks between containers
    """
    for network in NETWORKS:
        options = {
            "pipework": PIPEWORK_CMD,
        }
        for idx, node in enumerate(network.get("nodes")):
            options["node"] = node
            options["ip"] = idx+1
            options.update(network)
            cmd = "%(pipework)s br%(id)s -i eth%(id)s  %(node)s  192.168.%(id)s.%(ip)s/24" % options
            if onlyfornode is not None:
                if node == onlyfornode:
                    local(cmd)
            else:
                local(cmd)

@task
def ssh(name):
     """
     Open a ssh session to qqch@{container_name}
     """

     found = False
     for node in NODES:
          if node.get("name") == name:
              local("ssh qqch@127.1 -p220%(id)s " % node)
              found = True
     if not found:
          puts(red("%s not found in cluster definition" % name))

@task
def ls():
    """
    List containers configuration
    """
    for node in NODES:
        puts(yellow("%s : " % node.get("name") ))
        puts("------------------------")
        pprint(node)
        puts("------------------------\n\n")

### MYSQL PART

from tissu.tasks import *

import os, sys

current_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_dir)

def run_mysql(query, shell=False):
    from tissu.conf import settings

    node = node_find(settings.NODE_NAME)
    options = {"query": query}
    options.update(settings.DB)

    if shell:
        cmd = "mysql --user=%(user)s --host=%(host)s -p%(password)s" % options
    else:
        cmd = "mysql --user=%(user)s --host=%(host)s -p%(password)s -e \"%(query)s\"" % options
    run(cmd)

@roles('mysql')
@task
def mysql_shell():
    run_mysql("", shell=True)

@roles('mysql')
@task
def mysql_user_create(login, password,host="%"):
    sql="CREATE USER '%s'@'%s' IDENTIFIED BY '%s'" % (login,host,password)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_user_delete(login):
    drop = "DROP USER %s" % (login,)
    run_mysql(drop)

@roles('mysql')
@task
def mysql_database_create(name):
    sql="CREATE DATABASE %s" % (name,)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_privileges_add(login,database,host="localhost"):
    """
    login, database, host=localhost
    """
    sql = "GRANT ALL PRIVILEGES ON %s.* TO '%s'@'%s'" % (database,login,host)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_privileges_delete(login,database,host="localhost"):
    sql ="REVOKE ALL PRIVILEGES ON %s.* FROM '%s'@'%s'" % (database,login,host)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_privileges_show(login,host="localhost"):
    sql = "SHOW GRANTS FOR %s@%s" % (login,host)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_account_create(login,password):
    """
    login password
    """
    mysql_user_create(login,password)
    mysql_database_create(login)
    mysql_privileges_add(login,login)

<commit_msg>update mysql default host convention<commit_after>#-*- coding:utf8 -*-
# @author Thierry Stiegler thierry.stiegler@gmail.com

from fabric.api import *
from fabric.colors import red,yellow,green
from fabric.contrib.console import confirm
from fabric.utils import abort,error

from pprint import pprint

import os

# SETTINGS

IMAGES = ["base","postgresql", "mysql", "http", "php", "wsgi"]

NODES = [

## PROD CONTAINERS
    { "id": 1, "name" : "prodpostgresql01"     , "image":"qqch/postgresql:latest", "extra_port":"5432%(id)s:5432"  },
    { "id": 2, "name" : "prodmysql01"          , "image":"qqch/mysql:latest"     , "extra_port":"3306%(id)s:3306"  },
    { "id": 3, "name" : "prodjibaku01"         , "image":"qqch/wsgi:latest"      , "extra_port":"800%(id)s:80"     },
    { "id": 4, "name" : "prodjeanglode01"      , "image":"qqch/wsgi:latest"      , "extra_port":"800%(id)s:80"     },
    { "id": 5, "name" : "prodjeanglode02"      , "image":"qqch/php:latest"       , "extra_port":"800%(id)s:80"     },
    { "id": 6, "name" : "prodwp01"             , "image":"qqch/php:latest"       , "extra_port":"800%(id)s:80"     },
    { "id": 7, "name" : "prodtorzka01"         , "image":"qqch/php:latest"       , "extra_port":"800%(id)s:80"     },
    { "id": 8, "name" : "prodnolann01"         , "image":"qqch/http:latest"       , "extra_port":"800%(id)s:80"     },

## TEST CONTAINERS
    { "id": 9, "name" : "jouet01"         , "image":"qqch/base:latest"       ,  },
]

NETWORKS = [
    { "id": 1, "nodes": ("prodmysql01", "prodjeanglode02", "prodjeanglode01") },
    { "id": 2, "nodes": ("prodpostgresql01", "prodjibaku01")},
    { "id": 3, "nodes": ("prodmysql01", "prodwp01")},
    { "id": 4, "nodes": ("prodmysql01", "prodtorzka01")},

]

DOCKERFILES_ROOT = "/home/docker/dockerfiles"
CONTAINERS_DATAROOT = "/home/docker/volumes/"

DKCLEAN_CMD = "/home/docker/bin/dkclean.sh"
PIPEWORK_CMD = "sudo /home/docker/bin/pipework"
# TASKS

@task
def node(nodename,cmd=None):
    """
    The node mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab node:{nodename},boot : do a 'docker.io run' on each container, run it once time
      * fab node:{nodename},stop : stop all containers in the node
      * fab node:{nodename},start : do a 'docker.io start' on each container
      * fab node:{nodename},destroy : destroy all the node and clean docker's data (see dkclean)
    """

    generic_msg  = "\nAvailable commands are :\n  * start\n  * stop\n  * destroy\n  * boot"

    if cmd == "start":
        node_start(nodename)
        puts(green("node start [OK]"))
    elif cmd == "stop":
        node_stop(nodename)
        puts(green("node stop [OK]"))
    elif cmd == "boot":
        node_boot(nodename)
        puts(green("node boot [OK]"))
    elif cmd == "destroy":
        if confirm("Are you sure to destroy ? Everything you need is backuped ?",default=False):
            node_destroy(nodename)
            puts(green("node destroy [OK]"))
        else:
            abort(yellow("node destroy [CANCELED]"))
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

@task
def cluster(cmd=None):
    """
    The cluster mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab cluster:boot : do a 'docker.io run' on each container, run it once time
      * fab cluster:stop : stop all containers in the cluster
      * fab cluster:start : do a 'docker.io start' on each container
      * fab cluster:destroy : destroy all the cluster and clean docker's data (see dkclean)
    """

    generic_msg  = "\nAvailable commands are :\n  * start\n  * stop\n  * destroy\n  * boot"

    if cmd == "start":
        cluster_start()
        puts(green("cluster start [OK]"))
    elif cmd == "stop":
        cluster_stop()
        puts(green("cluster stop [OK]"))
    elif cmd == "boot":
        cluster_boot()
        puts(green("cluster boot [OK]"))
    elif cmd == "destroy":
        if confirm("Are you sure to destroy ? Everything you need is backuped ?",default=False):
            cluster_destroy()
            puts(green("cluster destroy [OK]"))
        else:
            abort(yellow("cluster destroy [CANCELED]"))
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

@task
def dockerui(cmd=None):
    """
    The dockerui mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab dockerui:boot : do a 'docker.io run' on each container, run it once time
      * fab dockerui:stop : stop all containers in the dockerui
      * fab dockerui:start : do a 'docker.io start' on each container
      * fab dockerui:destroy : destroy all the dockerui and clean docker's data (see dkclean)
    """

    generic_msg  = "\nAvailable commands are :\n  * start\n  * stop\n  * destroy\n  * boot"

    if cmd == "start":
        dockerui_start()
        puts(green("dockerui start [OK]"))
    elif cmd == "stop":
        dockerui_stop()
        puts(green("dockerui stop [OK]"))
    elif cmd == "boot":
        dockerui_boot()
        puts(green("dockerui boot [OK]"))
    elif cmd == "destroy":
        if confirm("Are you sure to destroy ? Everything you need is backuped ?",default=False):
            dockerui_destroy()
            puts(green("dockerui destroy [OK]"))
        else:
            abort(yellow("dockerui destroy [CANCELED]"))
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

@task
def build(*images):
    """
    Build given images in args. if args is empty it builds %s
    """ % IMAGES

    if len(images) <= 0:
        images = IMAGES

    for image in images:
        dirname = "qqch-%s" % (image,)
        options = {
          "image":image,
          "dpath" : os.path.join(DOCKERFILES_ROOT, dirname),
         }
        local("docker.io build -t qqch/%(image)s %(dpath)s" % options)
    dkclean()

@task
def dkclean():
    """
    Warn the user to run the dkclean
    """
    warn(yellow("Please run : %s" % DKCLEAN_CMD))

def dockerui_boot():
    local("docker.io run -h dockerui --name=dockerui -d -p 9000:9000 -v /var/run/docker.sock:/docker.sock crosbymichael/dockerui -e /docker.sock")

def dockerui_start():
    local("docker.io start dockerui")

def dockerui_stop():
    local("docker.io stop dockerui")

def dockerui_destroy():
    local("docker.io kill dockerui")
    dkclean()

def getports(defnode):
    options = {
      "convention" : "--publish=220%(id)s:22 --publish=490%(id)s:4949" ,
      "extra"      : "",
    }

    if defnode.has_key("extra_port"):
        options["extra"] = "--publish=%(extra_port)s" % defnode

    tpl = "%(convention)s %(extra)s" % options
    return tpl % defnode

def getvpath(defnode):
    return os.path.join(CONTAINERS_DATAROOT, defnode.get("name"))

def getlinks(defnode):
    r = ""
    for link in defnode.get("links", [] ):
        fragment = "--link=%s" % link
        r = r + fragment
    return r

CMD_BOOT = "docker.io run --hostname=%(name)s --name=%(name)s -v %(vpath)s:/data %(ports)s %(linking)s -d -t %(image)s  /sbin/my_init"
def cluster_boot():
    for defnode in NODES:
        defnode["ports"] = getports(defnode)
        defnode["vpath"] = getvpath(defnode)
        defnode["linking"] = getlinks(defnode)
        local("mkdir -p %(vpath)s" % defnode)
        cmd = CMD_BOOT % defnode
        local(cmd)
    network_up()

def node_find(nodename):
     """
     Search a node by it's name, return None if not found
     """
     for node in NODES:
          if node.get("name") == nodename:
              found = True
              return node

     puts(red("%s not found in cluster definition" % nodename))
     return None

def node_boot(nodename):
    defnode = node_find(nodename)
    if defnode:
        defnode["ports"] = getports(defnode)
        defnode["vpath"] = getvpath(defnode)
        defnode["linking"] = getlinks(defnode)
        local(CMD_BOOT % defnode)
        network_up(nodename)

def cluster_exec(cmd):
    for defnode in NODES :
        local(cmd % defnode)

def node_exec(nodename,cmd):
    defnode = node_find(nodename)
    if defnode:
        local(cmd % defnode)

CMD_STOP = "docker.io stop %(name)s"
def cluster_stop():
    cluster_exec(CMD_STOP)

def node_stop(nodename):
    node_exec(nodename, CMD_STOP)

CMD_DESTROY = "docker.io kill %(name)s"
def cluster_destroy():
    cluster_exec(CMD_DESTROY)
    dkclean()

def node_destroy(nodename):
    node_exec(nodename, CMD_DESTROY)
    dkclean()

CMD_START = "docker.io start %(name)s"
def cluster_start():
    cluster_exec(CMD_START)
    network_up()

def node_start(nodename):
    node_exec(nodename, CMD_START)
    network_up(nodename)

def conf_render(tpl):
    for defnode in NODES:
        puts(tpl % defnode)

@task
def conf(cmd=None):
    """
    The conf mini wrapper, values for cmd arg : boot,start,stop,destroy
      * fab conf:munin : generate the node tree for /etc/munin/munin.conf
      * fab conf:upstreams : generate the upstreams definition for nginx
      * fab conf:ufw : generate the ufw rules for making ssh available from the outside for every container
    """

    generic_msg  = "\nAvailable commands are :\n  * munin\n  * ssh"

    if cmd == "ssh":
        conf_munin()
    elif cmd == "upstreams":
        conf_upstreams()
    elif cmd == "ufw":
        puts(generic_msg)
    elif cmd == None:
        puts(generic_msg)
    else:
        error(red("commande %s not found !" % cmd))
        puts(generic_msg)

def conf_munin():
    tpl = """
[docker;%(name)s]
    address 172.17.42.1
    port 490%(id)s
    use_node_name yes
"""
    conf_render(tpl)

def conf_upstreams():
    tpl ="""
upstream %(name)s {
    server 172.17.42.1:800%(id)s;
}
"""
    conf_render(tpl)

def conf_ufw():
    tpl = """
ufw allow out 220%(id)s
"""
    conf_render(tpl)

@task
def network_up(onlyfornode=None):
    """
    Build privates networks between containers
    """
    for network in NETWORKS:
        options = {
            "pipework": PIPEWORK_CMD,
        }
        for idx, node in enumerate(network.get("nodes")):
            options["node"] = node
            options["ip"] = idx+1
            options.update(network)
            cmd = "%(pipework)s br%(id)s -i eth%(id)s  %(node)s  192.168.%(id)s.%(ip)s/24" % options
            if onlyfornode is not None:
                if node == onlyfornode:
                    local(cmd)
            else:
                local(cmd)

@task
def ssh(name):
     """
     Open a ssh session to qqch@{container_name}
     """

     found = False
     for node in NODES:
          if node.get("name") == name:
              local("ssh qqch@127.1 -p220%(id)s " % node)
              found = True
     if not found:
          puts(red("%s not found in cluster definition" % name))

@task
def ls():
    """
    List containers configuration
    """
    for node in NODES:
        puts(yellow("%s : " % node.get("name") ))
        puts("------------------------")
        pprint(node)
        puts("------------------------\n\n")

### MYSQL PART

from tissu.tasks import *

import os, sys

current_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_dir)

def run_mysql(query, shell=False):
    from tissu.conf import settings

    node = node_find(settings.NODE_NAME)
    options = {"query": query}
    options.update(settings.DB)

    if shell:
        cmd = "mysql --user=%(user)s --host=%(host)s -p%(password)s" % options
    else:
        cmd = "mysql --user=%(user)s --host=%(host)s -p%(password)s -e \"%(query)s\"" % options
    run(cmd)

@roles('mysql')
@task
def mysql_shell():
    run_mysql("", shell=True)

@roles('mysql')
@task
def mysql_user_create(login, password,host="%"):
    sql="CREATE USER '%s'@'%s' IDENTIFIED BY '%s'" % (login,host,password)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_user_delete(login):
    drop = "DROP USER %s" % (login,)
    run_mysql(drop)

@roles('mysql')
@task
def mysql_database_create(name):
    sql="CREATE DATABASE %s" % (name,)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_database_delete(name):
    sql ="DROP DATABASE %s" % (name,)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_privileges_add(login,database,host="%"):
    """
    login, database, host=localhost
    """
    sql = "GRANT ALL PRIVILEGES ON %s.* TO '%s'@'%s'" % (database,login,host)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_privileges_delete(login,database,host="%"):
    sql ="REVOKE ALL PRIVILEGES ON %s.* FROM '%s'@'%s'" % (database,login,host)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_privileges_show(login,host="%"):
    sql = "SHOW GRANTS FOR %s@%s" % (login,host)
    run_mysql(sql)

@roles('mysql')
@task
def mysql_account_create(login,password):
    """
    login password
    """
    mysql_user_create(login,password)
    mysql_database_create(login)
    mysql_privileges_add(login,login)

<|endoftext|>


<commit_before>from zope.interface import implementer
from Acquisition import aq_parent
from AccessControl import ClassSecurityInfo
from DateTime import DateTime
from Products.Archetypes import atapi
from Products.CMFCore.permissions import ModifyPortalContent
from Products.CMFDynamicViewFTI.browserdefault import BrowserDefaultMixin
from plone.app.event.base import DT
from plone.formwidget.datetime.at import DatetimeWidget
from archetypes.referencebrowserwidget import ReferenceBrowserWidget
from bda.plone.shop.interfaces import IBuyable
from ..interfaces import ITicketOccurrence
from ..config import PROJECTNAME
from .. import _

schema = atapi.Schema((

    atapi.ReferenceField('canonical_ticket',
        widget=ReferenceBrowserWidget(
            label=_(u'label_canonical_ticket', default=u'Canonical Ticket'),
        ),
        write_permission=ModifyPortalContent,
        allowed_types=['Ticket'],
        multiValued=0,
        relationship='canonical_ticket',
    ),

    atapi.DateTimeField('startDate',
        required=True,
        searchable=False,
        accessor='start',
        write_permission=ModifyPortalContent,
        languageIndependent=True,
        widget=DatetimeWidget(
            label=_(u'label_start', default=u'Start'),
            description=_(u'help_start',
                          default=u"Date and Time"),
            with_time=1,
            ),
        ),

    atapi.DateTimeField('endDate',
        required=True,
        searchable=False,
        accessor='end',
        write_permission=ModifyPortalContent,
        languageIndependent=True,
        widget=DatetimeWidget(
            label=_(u'label_end', default=u'End'),
            description=_(u'help_end',
                          default=u"Date and Time"),
            with_time=1,
            ),
        ),
))

TicketOccurrence_schema = atapi.BaseFolderSchema.copy() + schema.copy()

@implementer(ITicketOccurrence, IBuyable)
class TicketOccurrence(atapi.BaseFolder, BrowserDefaultMixin):
    security = ClassSecurityInfo()
    portal_type = 'Ticket Occurrence'
    meta_type = 'TicketOccurrence'
    _at_rename_after_creation = True
    schema = TicketOccurrence_schema
    exclude_from_nav = True

    def _dt_getter(self, field):
        event = aq_parent(self)
        timezone = event.getField('timezone').get(event)
        dt = self.getField(field).get(self)
        return dt.toZone(timezone)

    def _dt_setter(self, fieldtoset, value, **kwargs):
        if not isinstance(value, DateTime):
            value = DT(value)
        value = DateTime(
            '%04d-%02d-%02dT%02d:%02d:%02d' % (
                value.year(),
                value.month(),
                value.day(),
                value.hour(),
                value.minute(),
                int(value.second())
            )
        )
        self.getField(fieldtoset).set(self, value, **kwargs)

    security.declareProtected('View', 'start')
    def start(self):
        return self._dt_getter('startDate')

    security.declareProtected('View', 'end')
    def end(self):
        return self._dt_getter('endDate')

    security.declareProtected(ModifyPortalContent, 'setStartDate')
    def setStartDate(self, value, **kwargs):
        self._dt_setter('startDate', value, **kwargs)

    security.declareProtected(ModifyPortalContent, 'setEndDate')
    def setEndDate(self, value, **kwargs):
        self._dt_setter('endDate', value, **kwargs)

atapi.registerType(TicketOccurrence, PROJECTNAME)
<commit_msg>ticket occurrences are children of ticket, no reference needed<commit_after>from zope.interface import implementer
from Acquisition import aq_parent
from AccessControl import ClassSecurityInfo
from DateTime import DateTime
from Products.Archetypes import atapi
from Products.CMFCore.permissions import ModifyPortalContent
from Products.CMFDynamicViewFTI.browserdefault import BrowserDefaultMixin
from plone.app.event.base import DT
from plone.formwidget.datetime.at import DatetimeWidget
from bda.plone.shop.interfaces import IBuyable
from ..interfaces import ITicketOccurrence
from ..config import PROJECTNAME
from .. import _

schema = atapi.Schema((

    atapi.DateTimeField('startDate',
        required=True,
        searchable=False,
        accessor='start',
        write_permission=ModifyPortalContent,
        languageIndependent=True,
        widget=DatetimeWidget(
            label=_(u'label_start', default=u'Start'),
            description=_(u'help_start',
                          default=u"Date and Time"),
            with_time=1,
            ),
        ),

    atapi.DateTimeField('endDate',
        required=True,
        searchable=False,
        accessor='end',
        write_permission=ModifyPortalContent,
        languageIndependent=True,
        widget=DatetimeWidget(
            label=_(u'label_end', default=u'End'),
            description=_(u'help_end',
                          default=u"Date and Time"),
            with_time=1,
            ),
        ),
))

TicketOccurrence_schema = atapi.BaseFolderSchema.copy() + schema.copy()

@implementer(ITicketOccurrence, IBuyable)
class TicketOccurrence(atapi.BaseFolder, BrowserDefaultMixin):
    security = ClassSecurityInfo()
    portal_type = 'Ticket Occurrence'
    meta_type = 'TicketOccurrence'
    _at_rename_after_creation = True
    schema = TicketOccurrence_schema
    exclude_from_nav = True

    def _dt_getter(self, field):
        event = aq_parent(aq_parent(self))
        timezone = event.getField('timezone').get(event)
        dt = self.getField(field).get(self)
        return dt.toZone(timezone)

    def _dt_setter(self, fieldtoset, value, **kwargs):
        if not isinstance(value, DateTime):
            value = DT(value)
        value = DateTime(
            '%04d-%02d-%02dT%02d:%02d:%02d' % (
                value.year(),
                value.month(),
                value.day(),
                value.hour(),
                value.minute(),
                int(value.second())
            )
        )
        self.getField(fieldtoset).set(self, value, **kwargs)

    security.declareProtected('View', 'start')
    def start(self):
        return self._dt_getter('startDate')

    security.declareProtected('View', 'end')
    def end(self):
        return self._dt_getter('endDate')

    security.declareProtected(ModifyPortalContent, 'setStartDate')
    def setStartDate(self, value, **kwargs):
        self._dt_setter('startDate', value, **kwargs)

    security.declareProtected(ModifyPortalContent, 'setEndDate')
    def setEndDate(self, value, **kwargs):
        self._dt_setter('endDate', value, **kwargs)

atapi.registerType(TicketOccurrence, PROJECTNAME)
<|endoftext|>


<commit_before>"""
Collect memcached slab stats

#### Example Configuration

[[MemcachedSlabCollector]]
enabled = True
host = localhost  # optional
port = 11211  # optional
"""

from collections import defaultdict
import socket

import diamond.collector
from diamond.metric import Metric

def parse_slab_stats(slab_stats):
    """Convert output from memcached's `stats slabs` into a Python dict.

    Newlines are returned by memcached along with carriage returns
    (i.e. '\r\n').

    >>> parse_slab_stats("STAT 1:chunk_size 96\r\nSTAT 1:chunks_per_page 10922\r\nSTAT active_slabs 1\r\nSTAT total_malloced 1048512\r\nEND\r\n")
    {
        'slabs': {
            1: {
                'chunk_size': 96,
                'chunks_per_page': 10922,
                # ...
            },
        },
        'active_slabs': 1,
        'total_malloced': 1048512,
    }
    """
    stats_dict = {}
    stats_dict['slabs'] = defaultdict(lambda: {})

    for line in slab_stats.splitlines():
        if line == 'END':
            break
        # e.g.: "STAT 1:chunks_per_page 10922"
        cmd, key, value = line.split(' ')
        if cmd != 'STAT':
            continue
        # e.g.: "STAT active_slabs 1"
        if ":" not in key:
            stats_dict[key] = int(value)
            continue
        slab, key = key.split(':')
        stats_dict['slabs'][int(slab)][key] = int(value)

    return stats_dict

def dict_to_paths(dict_):
    """Convert a dict to metric paths.

    >>> dict_to_paths({'foo': {'bar': 1}, 'baz': 2})
    {
        'foo.bar': 1,
        'baz': 2,
    }
    """
    metrics = {}
    for k, v in dict_.iteritems():
        if isinstance(v, dict):
            submetrics = dict_to_paths(v)
            for subk, subv in submetrics.iteritems():
                metrics['.'.join([str(k), str(subk)])] = subv
        else:
            metrics[k] = v
    return metrics

class MemcachedSlabCollector(diamond.collector.Collector):

    def process_config(self):
        super(MemcachedSlabCollector, self).process_config()
        self.host = self.config['host']
        self.port = int(self.config['port'])

    def get_default_config(self):
        config = super(MemcachedSlabCollector, self).get_default_config()
        # Output stats in the format:
        # 'servers.cache-main-01.memcached_slab.slabs.1.chunk_size'
        config.update({
            'interval': 60,
            'path_prefix': 'servers',
            'path': 'memcached_slab',
            'host': 'localhost',
            'port': 11211,
        })
        return config

    def get_slab_stats(self):
        """Retrieve slab stats from memcached."""
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((self.host, self.port))
        s.send("stats slabs\n")
        try:
            data = ""
            while True:
                data += s.recv(4096)
                if data.endswith('END\r\n'):
                    break
            return data
        finally:
            s.close()

    def collect(self):
        unparsed_slab_stats = self.get_slab_stats()
        slab_stats = parse_slab_stats(unparsed_slab_stats)
        paths = dict_to_paths(slab_stats)
        for path, value in paths.iteritems():
            # Add path and prefix to metric (e.g.
            # 'servers.cache-main-01.memchached_slab')
            full_path = self.get_metric_path(path)
            metric = Metric(full_path, value)
            self.publish_metric(metric)
<commit_msg>Fix pep8 E501.<commit_after>"""
Collect memcached slab stats

#### Example Configuration

[[MemcachedSlabCollector]]
enabled = True
host = localhost  # optional
port = 11211  # optional
"""

from collections import defaultdict
import socket

import diamond.collector
from diamond.metric import Metric

def parse_slab_stats(slab_stats):
    """Convert output from memcached's `stats slabs` into a Python dict.

    Newlines are returned by memcached along with carriage returns
    (i.e. '\r\n').

    >>> parse_slab_stats(
            "STAT 1:chunk_size 96\r\nSTAT 1:chunks_per_page 10922\r\nSTAT "
            "active_slabs 1\r\nSTAT total_malloced 1048512\r\nEND\r\n")
    {
        'slabs': {
            1: {
                'chunk_size': 96,
                'chunks_per_page': 10922,
                # ...
            },
        },
        'active_slabs': 1,
        'total_malloced': 1048512,
    }
    """
    stats_dict = {}
    stats_dict['slabs'] = defaultdict(lambda: {})

    for line in slab_stats.splitlines():
        if line == 'END':
            break
        # e.g.: "STAT 1:chunks_per_page 10922"
        cmd, key, value = line.split(' ')
        if cmd != 'STAT':
            continue
        # e.g.: "STAT active_slabs 1"
        if ":" not in key:
            stats_dict[key] = int(value)
            continue
        slab, key = key.split(':')
        stats_dict['slabs'][int(slab)][key] = int(value)

    return stats_dict

def dict_to_paths(dict_):
    """Convert a dict to metric paths.

    >>> dict_to_paths({'foo': {'bar': 1}, 'baz': 2})
    {
        'foo.bar': 1,
        'baz': 2,
    }
    """
    metrics = {}
    for k, v in dict_.iteritems():
        if isinstance(v, dict):
            submetrics = dict_to_paths(v)
            for subk, subv in submetrics.iteritems():
                metrics['.'.join([str(k), str(subk)])] = subv
        else:
            metrics[k] = v
    return metrics

class MemcachedSlabCollector(diamond.collector.Collector):

    def process_config(self):
        super(MemcachedSlabCollector, self).process_config()
        self.host = self.config['host']
        self.port = int(self.config['port'])

    def get_default_config(self):
        config = super(MemcachedSlabCollector, self).get_default_config()
        # Output stats in the format:
        # 'servers.cache-main-01.memcached_slab.slabs.1.chunk_size'
        config.update({
            'interval': 60,
            'path_prefix': 'servers',
            'path': 'memcached_slab',
            'host': 'localhost',
            'port': 11211,
        })
        return config

    def get_slab_stats(self):
        """Retrieve slab stats from memcached."""
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((self.host, self.port))
        s.send("stats slabs\n")
        try:
            data = ""
            while True:
                data += s.recv(4096)
                if data.endswith('END\r\n'):
                    break
            return data
        finally:
            s.close()

    def collect(self):
        unparsed_slab_stats = self.get_slab_stats()
        slab_stats = parse_slab_stats(unparsed_slab_stats)
        paths = dict_to_paths(slab_stats)
        for path, value in paths.iteritems():
            # Add path and prefix to metric (e.g.
            # 'servers.cache-main-01.memchached_slab')
            full_path = self.get_metric_path(path)
            metric = Metric(full_path, value)
            self.publish_metric(metric)
<|endoftext|>


<commit_before>"""
fs.path
=======

Useful functions for FS path manipulation.

This is broadly similar to the standard ``os.path`` module but works with
paths in the canonical format expected by all FS objects (forwardslash-separated,
optional leading slash).

"""

def normpath(path):
    """Normalizes a path to be in the format expected by FS objects.

    This function remove any leading or trailing slashes, collapses
    duplicate slashes, replaces forward with backward slashes, and generally
    tries very hard to return a new path string the canonical FS format.
    If the path is invalid, ValueError will be raised.

    :param path: path to normalize
    :returns: a valid FS path

    >>> normpath(r"foo\\bar\\baz")
    'foo/bar/baz'

    >>> normpath("/foo//bar/frob/../baz")
    '/foo/bar/baz'

    >>> normpath("foo/../../bar")
    Traceback (most recent call last)
        ...
    ValueError: too many backrefs in path 'foo/../../bar'

    """
    if not path:
        return path
    components = []
    append = components.append
    for comp in [c for c in path.replace('\\','/').split("/") if c not in ('', '.')]:
        if comp == "..":
            try:
                components.pop()
            except IndexError:
                err = "too many backrefs in path '%s'" % (path,)
                raise ValueError(err)
        else:
            append(comp)
    if path[0] in '\\/':
        if not components:
            append("")
        components.insert(0, "")
    return "/".join(components)

def iteratepath(path, numsplits=None):
    """Iterate over the individual components of a path.

    :param path: Path to iterate over
    :numsplits: Maximum number of splits

    """
    path = relpath(normpath(path))
    if not path:
        return []
    if numsplits == None:
        return path.split('/')
    else:
        return path.split('/', numsplits)

def recursepath(path, reverse=False):
    """Returns intermediate paths from the root to the given path

    :param reverse: reverses the order of the paths

    >>> recursepath('a/b/c')
    ['/', u'/a', u'/a/b', u'/a/b/c']

    """
    paths = [u'/']
    append = paths.append
    path = u'/' + normpath(path.lstrip('/'))
    find = path.find
    pos = 1
    if len(path) > 1:
        while 1:
            pos = find('/', pos + 1)
            if pos == -1:
                append(path)
                break
            append(path[:pos])

    if reverse:
        return paths[::-1]
    return paths

def abspath(path):
    """Convert the given path to an absolute path.

    Since FS objects have no concept of a 'current directory' this simply
    adds a leading '/' character if the path doesn't already have one.

    """
    if not path.startswith('/'):
        return u'/' + path
    return path

def relpath(path):
    """Convert the given path to a relative path.

    This is the inverse of abspath(), stripping a leading '/' from the
    path if it is present.

    :param path: Path to adjust

    >>> relpath('/a/b')
    'a/b'

    """
    return path.lstrip('/')

def pathjoin(*paths):
    """Joins any number of paths together, returning a new path string.

    :param paths: Paths to join are given in positional arguments

    >>> pathjoin('foo', 'bar', 'baz')
    'foo/bar/baz'

    >>> pathjoin('foo/bar', '../baz')
    'foo/baz'

    >>> pathjoin('foo/bar', '/baz')
    '/baz'

    """
    absolute = False
    relpaths = []
    for p in paths:
        if p:
             if p[0] in '\\/':
                 del relpaths[:]
                 absolute = True
             relpaths.append(p)

    path = normpath(u"/".join(relpaths))
    if absolute:
        path = abspath(path)
    return path

# Allow pathjoin() to be used as fs.path.join()
join = pathjoin

def pathsplit(path):
    """Splits a path into (head, tail) pair.

    This function splits a path into a pair (head, tail) where 'tail' is the
    last pathname component and 'head' is all preceeding components.

    :param path: Path to split

    >>> pathsplit("foo/bar")
    ('foo', 'bar')

    >>> pathsplit("foo/bar/baz")
    ('foo/bar', 'baz')

    >>> pathsplit("/foo/bar/baz")
    ('/foo/bar', 'baz')

    """
    if '/' not in path:
        return ('', path)
    split = path.rsplit('/', 1)
    return (split[0] or '/', split[1])

# Allow pathsplit() to be used as fs.path.split()
split = pathsplit

def splitext(path):
    """Splits the extension from the path, and returns the path (up to the last
    '.' and the extension

    :param path: A path to split

    >>> splitext('baz.txt')
    ('baz', 'txt')

    >>> splitext('foo/bar/baz.txt')
    ('foo/bar/baz', 'txt')

    """

    parent_path, pathname = pathsplit(path)
    if '.' not in pathname:
        return path, ''
    pathname, ext = pathname.rsplit('.', 1)
    path = pathjoin(parent_path, pathname)
    return path, '.' + ext

def isdotfile(path):
    """Detects if a path references a dot file, i.e. a resource who's name
    starts with a '.'

    :param path: Path to check

    >>> isdotfile('.baz')
    True

    >>> isdotfile('foo/bar/baz')
    True

    >>> isdotfile('foo/bar.baz').
    False

    """
    return basename(path).startswith('.')

def dirname(path):
    """Returns the parent directory of a path.

    This is always equivalent to the 'head' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> dirname('foo/bar/baz')
    'foo/bar'

    """
    if '/' not in path:
        return ''
    return path.rsplit('/', 1)[0]

def basename(path):
    """Returns the basename of the resource referenced by a path.

    This is always equivalent to the 'head' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> basename('foo/bar/baz')
    'baz'

    """
    if '/' not in path:
        return path
    return path.rsplit('/', 1)[-1]

def issamedir(path1, path2):
    """Return true if two paths reference a resource in the same directory.

    :param path1: An FS path
    :param path2: An FS path

    >>> issamedir("foo/bar/baz.txt", "foo/bar/spam.txt")
    True
    >>> issamedir("foo/bar/baz/txt", "spam/eggs/spam.txt")
    False

    """
    return dirname(normpath(path1)) == dirname(normpath(path2))

def isbase(path1, path2):
    p1 = forcedir(abspath(path1))
    p2 = forcedir(abspath(path2))
    return p1 == p2 or p1.startswith(p2)

def isprefix(path1, path2):
    """Return true is path1 is a prefix of path2.

    :param path1: An FS path
    :param path2: An FS path

    >>> isprefix("foo/bar", "foo/bar/spam.txt")
    True
    >>> isprefix("foo/bar/", "foo/bar")
    True
    >>> isprefix("foo/barry", "foo/baz/bar")
    False
    >>> isprefix("foo/bar/baz/", "foo/baz/bar")
    False

    """
    bits1 = path1.split("/")
    bits2 = path2.split("/")
    while bits1 and bits1[-1] == "":
        bits1.pop()
    if len(bits1) > len(bits2):
        return False
    for (bit1,bit2) in zip(bits1,bits2):
        if bit1 != bit2:
            return False
    return True

def forcedir(path):
    """Ensure the path ends with a trailing /

    :param path: An FS path

    >>> forcedir("foo/bar")
    'foo/bar/'
    >>> forcedir("foo/bar/")
    'foo/bar/'

    """

    if not path.endswith('/'):
        return path + '/'
    return path

def frombase(path1, path2):
    if not isprefix(path1, path2):
        raise ValueError("path1 must be a prefix of path2")
    return path2[len(path1):]

class PathMap(object):
    """Dict-like object with paths for keys.

    A PathMap is like a dictionary where the keys are all FS paths.  It allows
    various dictionary operations (e.g. listing values, clearing values) to
    be performed on a subset of the keys sharing some common prefix, e.g.::

        # list all values in the map
        pm.values()

        # list all values for paths starting with "/foo/bar"
        pm.values("/foo/bar")

    Under the hood, a PathMap is a trie-like structure where each level is
    indexed by path name component.  This allows lookups to be performed in
    O(number of path components) while permitting efficient prefix-based
    operations.
    """

    def __init__(self):
        self._map = {}

    def __getitem__(self,path):
        """Get the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                raise KeyError(path)
        try:
            return m[""]
        except KeyError:
            raise KeyError(path)

    def __contains__(self,path):
        """Check whether the given path has a value stored in the map."""
        try:
            self[path]
        except KeyError:
            return False
        else:
            return True

    def __setitem__(self,path,value):
        """Set the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        m[""] = value

    def __delitem__(self,path):
        """Delete the value stored under the given path."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                raise KeyError(path)
            else:
                ms[-2][1] = name
        try:
            del ms[-1][0][""]
        except KeyError:
            raise KeyError(path)
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]

    def get(self,path,default=None):
        """Get the value stored under the given path, or the given default."""
        try:
            return self[path]
        except KeyError:
            return default

    def pop(self,path,default=None):
        """Pop the value stored under the given path, or the given default."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                return default
            else:
                ms[-2][1] = name
        try:
            val = ms[-1][0].pop("")
        except KeyError:
            val = default
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]
        return val

    def setdefault(self,path,value):
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        return m.setdefault("",value)

    def clear(self,root="/"):
        """Clear all entries beginning with the given root path."""
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        m.clear()

    def iterkeys(self,root="/",m=None):
        """Iterate over all keys beginning with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield abspath(normpath(root))
            else:
                k = pathjoin(root,nm)
                for subk in self.iterkeys(k,subm):
                    yield subk

    def keys(self,root="/"):
        return list(self.iterkeys(root))

    def itervalues(self,root="/",m=None):
        """Iterate over all values whose keys begin with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield subm
            else:
                k = pathjoin(root,nm)
                for subv in self.itervalues(k,subm):
                    yield subv

    def values(self,root="/"):
        return list(self.itervalues(root))

    def iteritems(self,root="/",m=None):
        """Iterate over all (key,value) pairs beginning with the given root."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield (abspath(normpath(root)),subm)
            else:
                k = pathjoin(root,nm)
                for (subk,subv) in self.iteritems(k,subm):
                    yield (subk,subv)

    def items(self,root="/"):
        return list(self.iteritems(root))

    def iternames(self,root="/"):
        """Iterate over all names beneath the given root path.

        This is basically the equivalent of listdir() for a PathMap - it yields
        the next level of name components beneath the given path.
        """
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        for (nm,subm) in m.iteritems():
            if nm and subm:
                yield nm

    def names(self,root="/"):
        return list(self.iternames(root))

_wild_chars = frozenset('*?[]!{}')
def iswildcard(path):
    """Check if a path ends with a wildcard

    >>> is_wildcard('foo/bar/baz.*')
    True
    >>> is_wildcard('foo/bar')
    False

    """
    assert path is not None
    base_chars = frozenset(basename(path))
    return not base_chars.isdisjoint(_wild_chars)
<commit_msg>Some optimizations<commit_after>"""
fs.path
=======

Useful functions for FS path manipulation.

This is broadly similar to the standard ``os.path`` module but works with
paths in the canonical format expected by all FS objects (forwardslash-separated,
optional leading slash).

"""

import re

_requires_normalization = re.compile(r'/\.\.|\./|//|\\').search
def normpath(path):
    """Normalizes a path to be in the format expected by FS objects.

    This function remove any leading or trailing slashes, collapses
    duplicate slashes, replaces forward with backward slashes, and generally
    tries very hard to return a new path string the canonical FS format.
    If the path is invalid, ValueError will be raised.

    :param path: path to normalize
    :returns: a valid FS path

    >>> normpath(r"foo\\bar\\baz")
    'foo/bar/baz'

    >>> normpath("/foo//bar/frob/../baz")
    '/foo/bar/baz'

    >>> normpath("foo/../../bar")
    Traceback (most recent call last)
        ...
    ValueError: too many backrefs in path 'foo/../../bar'

    """

    if path in ('', '/'):
        return path

    # An early out if there is no need to normalize this paath
    if not _requires_normalization(path):
        return path.rstrip('/')

    components = []
    append = components.append
    for comp in [c for c in path.replace('\\','/').split("/") if c not in ('', '.')]:
        if comp == "..":
            try:
                components.pop()
            except IndexError:
                raise ValueError("too many backrefs in path '%s'" % path)
        else:
            append(comp)
    if path[0] in '\\/':
        if not components:
            append("")
        components.insert(0, "")
    return "/".join(components)

def iteratepath(path, numsplits=None):
    """Iterate over the individual components of a path.

    :param path: Path to iterate over
    :numsplits: Maximum number of splits

    """
    path = relpath(normpath(path))
    if not path:
        return []
    if numsplits == None:
        return path.split('/')
    else:
        return path.split('/', numsplits)

def recursepath(path, reverse=False):
    """Returns intermediate paths from the root to the given path

    :param reverse: reverses the order of the paths

    >>> recursepath('a/b/c')
    ['/', u'/a', u'/a/b', u'/a/b/c']

    """

    if path in ('', '/'):
        return [u'/']

    path = abspath(normpath(path)) + '/'

    paths = [u'/']
    find = path.find
    append = paths.append
    pos = 1
    len_path = len(path)

    while pos < len_path:
        pos = find('/', pos)
        append(path[:pos])
        pos += 1

    if reverse:
        return paths[::-1]
    return paths

def abspath(path):
    """Convert the given path to an absolute path.

    Since FS objects have no concept of a 'current directory' this simply
    adds a leading '/' character if the path doesn't already have one.

    """
    if not path.startswith('/'):
        return u'/' + path
    return path

def relpath(path):
    """Convert the given path to a relative path.

    This is the inverse of abspath(), stripping a leading '/' from the
    path if it is present.

    :param path: Path to adjust

    >>> relpath('/a/b')
    'a/b'

    """
    return path.lstrip('/')

def pathjoin(*paths):
    """Joins any number of paths together, returning a new path string.

    :param paths: Paths to join are given in positional arguments

    >>> pathjoin('foo', 'bar', 'baz')
    'foo/bar/baz'

    >>> pathjoin('foo/bar', '../baz')
    'foo/baz'

    >>> pathjoin('foo/bar', '/baz')
    '/baz'

    """
    absolute = False
    relpaths = []
    for p in paths:
        if p:
             if p[0] in '\\/':
                 del relpaths[:]
                 absolute = True
             relpaths.append(p)

    path = normpath(u"/".join(relpaths))
    if absolute:
        path = abspath(path)
    return path

# Allow pathjoin() to be used as fs.path.join()
join = pathjoin

def pathsplit(path):
    """Splits a path into (head, tail) pair.

    This function splits a path into a pair (head, tail) where 'tail' is the
    last pathname component and 'head' is all preceeding components.

    :param path: Path to split

    >>> pathsplit("foo/bar")
    ('foo', 'bar')

    >>> pathsplit("foo/bar/baz")
    ('foo/bar', 'baz')

    >>> pathsplit("/foo/bar/baz")
    ('/foo/bar', 'baz')

    """
    if '/' not in path:
        return ('', path)
    split = path.rsplit('/', 1)
    return (split[0] or '/', split[1])

# Allow pathsplit() to be used as fs.path.split()
split = pathsplit

def splitext(path):
    """Splits the extension from the path, and returns the path (up to the last
    '.' and the extension

    :param path: A path to split

    >>> splitext('baz.txt')
    ('baz', 'txt')

    >>> splitext('foo/bar/baz.txt')
    ('foo/bar/baz', 'txt')

    """

    parent_path, pathname = pathsplit(path)
    if '.' not in pathname:
        return path, ''
    pathname, ext = pathname.rsplit('.', 1)
    path = pathjoin(parent_path, pathname)
    return path, '.' + ext

def isdotfile(path):
    """Detects if a path references a dot file, i.e. a resource who's name
    starts with a '.'

    :param path: Path to check

    >>> isdotfile('.baz')
    True

    >>> isdotfile('foo/bar/baz')
    True

    >>> isdotfile('foo/bar.baz').
    False

    """
    return basename(path).startswith('.')

def dirname(path):
    """Returns the parent directory of a path.

    This is always equivalent to the 'head' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> dirname('foo/bar/baz')
    'foo/bar'

    """
    if '/' not in path:
        return ''
    return path.rsplit('/', 1)[0]

def basename(path):
    """Returns the basename of the resource referenced by a path.

    This is always equivalent to the 'head' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> basename('foo/bar/baz')
    'baz'

    """
    if '/' not in path:
        return path
    return path.rsplit('/', 1)[-1]

def issamedir(path1, path2):
    """Return true if two paths reference a resource in the same directory.

    :param path1: An FS path
    :param path2: An FS path

    >>> issamedir("foo/bar/baz.txt", "foo/bar/spam.txt")
    True
    >>> issamedir("foo/bar/baz/txt", "spam/eggs/spam.txt")
    False

    """
    return dirname(normpath(path1)) == dirname(normpath(path2))

def isbase(path1, path2):
    p1 = forcedir(abspath(path1))
    p2 = forcedir(abspath(path2))
    return p1 == p2 or p1.startswith(p2)

def isprefix(path1, path2):
    """Return true is path1 is a prefix of path2.

    :param path1: An FS path
    :param path2: An FS path

    >>> isprefix("foo/bar", "foo/bar/spam.txt")
    True
    >>> isprefix("foo/bar/", "foo/bar")
    True
    >>> isprefix("foo/barry", "foo/baz/bar")
    False
    >>> isprefix("foo/bar/baz/", "foo/baz/bar")
    False

    """
    bits1 = path1.split("/")
    bits2 = path2.split("/")
    while bits1 and bits1[-1] == "":
        bits1.pop()
    if len(bits1) > len(bits2):
        return False
    for (bit1,bit2) in zip(bits1,bits2):
        if bit1 != bit2:
            return False
    return True

def forcedir(path):
    """Ensure the path ends with a trailing /

    :param path: An FS path

    >>> forcedir("foo/bar")
    'foo/bar/'
    >>> forcedir("foo/bar/")
    'foo/bar/'

    """

    if not path.endswith('/'):
        return path + '/'
    return path

def frombase(path1, path2):
    if not isprefix(path1, path2):
        raise ValueError("path1 must be a prefix of path2")
    return path2[len(path1):]

class PathMap(object):
    """Dict-like object with paths for keys.

    A PathMap is like a dictionary where the keys are all FS paths.  It allows
    various dictionary operations (e.g. listing values, clearing values) to
    be performed on a subset of the keys sharing some common prefix, e.g.::

        # list all values in the map
        pm.values()

        # list all values for paths starting with "/foo/bar"
        pm.values("/foo/bar")

    Under the hood, a PathMap is a trie-like structure where each level is
    indexed by path name component.  This allows lookups to be performed in
    O(number of path components) while permitting efficient prefix-based
    operations.
    """

    def __init__(self):
        self._map = {}

    def __getitem__(self,path):
        """Get the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                raise KeyError(path)
        try:
            return m[""]
        except KeyError:
            raise KeyError(path)

    def __contains__(self,path):
        """Check whether the given path has a value stored in the map."""
        try:
            self[path]
        except KeyError:
            return False
        else:
            return True

    def __setitem__(self,path,value):
        """Set the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        m[""] = value

    def __delitem__(self,path):
        """Delete the value stored under the given path."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                raise KeyError(path)
            else:
                ms[-2][1] = name
        try:
            del ms[-1][0][""]
        except KeyError:
            raise KeyError(path)
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]

    def get(self,path,default=None):
        """Get the value stored under the given path, or the given default."""
        try:
            return self[path]
        except KeyError:
            return default

    def pop(self,path,default=None):
        """Pop the value stored under the given path, or the given default."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                return default
            else:
                ms[-2][1] = name
        try:
            val = ms[-1][0].pop("")
        except KeyError:
            val = default
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]
        return val

    def setdefault(self,path,value):
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        return m.setdefault("",value)

    def clear(self,root="/"):
        """Clear all entries beginning with the given root path."""
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        m.clear()

    def iterkeys(self,root="/",m=None):
        """Iterate over all keys beginning with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield abspath(normpath(root))
            else:
                k = pathjoin(root,nm)
                for subk in self.iterkeys(k,subm):
                    yield subk

    def keys(self,root="/"):
        return list(self.iterkeys(root))

    def itervalues(self,root="/",m=None):
        """Iterate over all values whose keys begin with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield subm
            else:
                k = pathjoin(root,nm)
                for subv in self.itervalues(k,subm):
                    yield subv

    def values(self,root="/"):
        return list(self.itervalues(root))

    def iteritems(self,root="/",m=None):
        """Iterate over all (key,value) pairs beginning with the given root."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield (abspath(normpath(root)),subm)
            else:
                k = pathjoin(root,nm)
                for (subk,subv) in self.iteritems(k,subm):
                    yield (subk,subv)

    def items(self,root="/"):
        return list(self.iteritems(root))

    def iternames(self,root="/"):
        """Iterate over all names beneath the given root path.

        This is basically the equivalent of listdir() for a PathMap - it yields
        the next level of name components beneath the given path.
        """
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        for (nm,subm) in m.iteritems():
            if nm and subm:
                yield nm

    def names(self,root="/"):
        return list(self.iternames(root))

_wild_chars = frozenset('*?[]!{}')
def iswildcard(path):
    """Check if a path ends with a wildcard

    >>> is_wildcard('foo/bar/baz.*')
    True
    >>> is_wildcard('foo/bar')
    False

    """
    assert path is not None
    base_chars = frozenset(basename(path))
    return not base_chars.isdisjoint(_wild_chars)

if __name__ == "__main__":
    print recursepath('a/b/c')<|endoftext|>


<commit_before>
import json
import os
import stat
import sys
import itertools

VERSION = 2

IDX_NAME = 0
IDX_SIZE = 1
IDX_MTIME = 2
IDX_MODE = 3
IDX_UID = 4
IDX_GID = 5

# target for symbolic links
# child nodes for directories
# nothing for files
IDX_TARGET = 6

# include trailing slash
PATH = "/mnt/"

EXCLUDE = set([
    "/boot/",
    #"/dev/",
    "/lost+found/",
    #"/proc/",
    #"/run/",
    #"/sys/",
    #"/tmp/",
])

def onerror(oserror):
    raise oserror

rootdepth = PATH.count("/")
files = os.walk(PATH, onerror=onerror)
prevpath = []

mainroot = []
result = {
    "fsroot": mainroot,
    "version": VERSION,
    "size": 0,
}
rootstack = [mainroot]

def make_node(st, name):
    obj = [None] * 7

    obj[IDX_NAME] = name
    obj[IDX_SIZE] = st.st_size
    obj[IDX_MTIME] = int(st.st_mtime)
    obj[IDX_MODE] = int(st.st_mode)

    obj[IDX_UID] = st.st_uid
    obj[IDX_GID] = st.st_gid

    result["size"] += st.st_size

    # Missing:
    #     int(st.st_atime),
    #     int(st.st_ctime),

    return obj

for f in files:
    dirpath, dirnames, filenames = f
    pathparts = dirpath.split("/")
    pathparts = pathparts[rootdepth:]
    fullpath = "/%s/" % "/".join(pathparts)

    if fullpath in EXCLUDE:
        dirnames[:] = []
        continue

    depth = 0
    for this, prev in zip(pathparts, prevpath):
        if this != prev:
            break
        depth += 1

    for name in prevpath[depth:]:
        rootstack.pop()

    oldroot = rootstack[-1]

    assert len(pathparts[depth:]) == 1
    openname = pathparts[-1]

    if openname == "":
        root = mainroot
    else:
        root = []
        st = os.stat(dirpath)
        rootobj = make_node(st, openname)
        rootobj[IDX_TARGET] = root
        oldroot.append(rootobj)

    rootstack.append(root)

    for filename in itertools.chain(filenames, dirnames):
        absname = os.path.join(dirpath, filename)

        st = os.lstat(absname)
        isdir = stat.S_ISDIR(st.st_mode)
        islink = stat.S_ISLNK(st.st_mode)

        if isdir and not islink:
            continue

        obj = make_node(st, filename)

        if islink:
            target = os.readlink(absname)
            obj[IDX_TARGET] = target

        while obj[-1] is None:
            obj.pop()

        root.append(obj)

    prevpath = pathparts

json.dump(result, sys.stdout, check_circular=False, separators=(',', ':'))

<commit_msg>if __name__ == "__main__"<commit_after>
import json
import os
import stat
import sys
import itertools

VERSION = 2

IDX_NAME = 0
IDX_SIZE = 1
IDX_MTIME = 2
IDX_MODE = 3
IDX_UID = 4
IDX_GID = 5

# target for symbolic links
# child nodes for directories
# nothing for files
IDX_TARGET = 6

# include trailing slash
PATH = "/mnt/"

EXCLUDE = set([
    "/boot/",
    #"/dev/",
    "/lost+found/",
    #"/proc/",
    #"/run/",
    #"/sys/",
    #"/tmp/",
])

def main():
    def onerror(oserror):
        raise oserror

    rootdepth = PATH.count("/")
    files = os.walk(PATH, onerror=onerror)
    prevpath = []

    mainroot = []
    result = {
        "fsroot": mainroot,
        "version": VERSION,
        "size": 0,
    }
    rootstack = [mainroot]

    def make_node(st, name):
        obj = [None] * 7

        obj[IDX_NAME] = name
        obj[IDX_SIZE] = st.st_size
        obj[IDX_MTIME] = int(st.st_mtime)
        obj[IDX_MODE] = int(st.st_mode)

        obj[IDX_UID] = st.st_uid
        obj[IDX_GID] = st.st_gid

        result["size"] += st.st_size

        # Missing:
        #     int(st.st_atime),
        #     int(st.st_ctime),

        return obj

    for f in files:
        dirpath, dirnames, filenames = f
        pathparts = dirpath.split("/")
        pathparts = pathparts[rootdepth:]
        fullpath = "/%s/" % "/".join(pathparts)

        if fullpath in EXCLUDE:
            dirnames[:] = []
            continue

        depth = 0
        for this, prev in zip(pathparts, prevpath):
            if this != prev:
                break
            depth += 1

        for name in prevpath[depth:]:
            rootstack.pop()

        oldroot = rootstack[-1]

        assert len(pathparts[depth:]) == 1
        openname = pathparts[-1]

        if openname == "":
            root = mainroot
        else:
            root = []
            st = os.stat(dirpath)
            rootobj = make_node(st, openname)
            rootobj[IDX_TARGET] = root
            oldroot.append(rootobj)

        rootstack.append(root)

        for filename in itertools.chain(filenames, dirnames):
            absname = os.path.join(dirpath, filename)

            st = os.lstat(absname)
            isdir = stat.S_ISDIR(st.st_mode)
            islink = stat.S_ISLNK(st.st_mode)

            if isdir and not islink:
                continue

            obj = make_node(st, filename)

            if islink:
                target = os.readlink(absname)
                obj[IDX_TARGET] = target

            while obj[-1] is None:
                obj.pop()

            root.append(obj)

        prevpath = pathparts

    json.dump(result, sys.stdout, check_circular=False, separators=(',', ':'))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from libcomxml.core import XmlModel, XmlField
from base import Order

SUPPORTED_ORDERS = ['B11']

def is_supported(order_code):
    return order_code in SUPPORTED_ORDERS

class B11:

    def __init__(self, generic_values, payload):
        self.generic_values = generic_values
        self.order = Order(
            generic_values.get('id_pet'),
            generic_values.get('id_req'),
            generic_values.get('cnc'),

        )
        self.order.cnc.feed({'payload':B11Payload(payload)})
        # Load generic order with values

class B11Payload(XmlModel):

    def __init__(self, payload, drop_empty=False):
        self.payload = XmlField(
            'B11', attributes={
                'Order': payload.get('txx'),
                'Args': '',
                'Fini': payload.get('date_from'),
                'Ffin': payload.get('date_to'),
            })
        super(B11Payload, self).__init__('b11Payload', 'payload', drop_empty=drop_empty)
<commit_msg>Fixed import<commit_after>from libcomxml.core import XmlModel, XmlField
from primestg.order.base import Order

SUPPORTED_ORDERS = ['B11']

def is_supported(order_code):
    return order_code in SUPPORTED_ORDERS

class B11:

    def __init__(self, generic_values, payload):
        self.generic_values = generic_values
        self.order = Order(
            generic_values.get('id_pet'),
            generic_values.get('id_req'),
            generic_values.get('cnc'),

        )
        self.order.cnc.feed({'payload':B11Payload(payload)})
        # Load generic order with values

class B11Payload(XmlModel):

    def __init__(self, payload, drop_empty=False):
        self.payload = XmlField(
            'B11', attributes={
                'Order': payload.get('txx'),
                'Args': '',
                'Fini': payload.get('date_from'),
                'Ffin': payload.get('date_to'),
            })
        super(B11Payload, self).__init__('b11Payload', 'payload', drop_empty=drop_empty)
<|endoftext|>