<commit_before>#!/usr/bin/env python
import functools
import sys
import commands
import logging
import time
import re
import signal
import threading

FENCER_TAG = "zs::ministorage::fencer"
MANAGEMENT_TAG = "zs::ministorage::management"

MINI_FENCER_KEY = "/usr/local/zstack/mini_fencer.key"
PEER_USERNAME = ""
PEER_MGMT_ADDR = ""

FENCE_GW_RESULT = None
OUTDATE_PEER_RESULT = None

logger = logging.getLogger(__name__)

def set_timeout(num, callback):
    def wrap(func):
        def handle(signum, frame):
            raise RuntimeError

        def to_do(*args, **kwargs):
            try:
                signal.signal(signal.SIGALRM, handle)
                signal.alarm(num)
                r = func(*args, **kwargs)
                signal.alarm(0)
                return r
            except RuntimeError as e:
                callback()

        return to_do

    return wrap

def retry(times=3, sleep_time=1):
    def wrap(f):
        @functools.wraps(f)
        def inner(*args, **kwargs):
            last_err = None
            for i in range(0, times):
                try:
                    return f(*args, **kwargs)
                except Exception as e:
                    last_err = e
                    logger.error(e)
                    time.sleep(sleep_time)
            logger.error("The task failed,  detail:\n%s" % last_err)
        return inner
    return wrap

def getstatusoutput(c):
    start_time = time.time()
    r, o = commands.getstatusoutput(c)
    end_time = time.time()
    logger.debug("command:[%s], returnCode:[%s], output:[%s], spendTime:[%s] ms" % (c, r, o, (end_time - start_time) * 1000))
    return r, o

def getoutput(c):
    return getstatusoutput(c)[1]

@retry()
def run(c):
    r, o = getstatusoutput(c)
    if r != 0:
        raise Exception("run command[%s] failed[return code: %s, output: %s]" % (c, r, o))

def test_fencer(vg_name, resource_name):
    logger.debug("run test_fencer %s" % resource_name)
    global FENCE_GW_RESULT
    FENCE_GW_RESULT = do_test_fencer(vg_name)

def do_test_fencer(vg_name):
    # type: (str) -> bool
    r, fencer_ip = getstatusoutput("timeout 3 vgs %s -otags --nolocking --noheading | tr ',' '\\n' | grep %s" % (vg_name, FENCER_TAG))
    if r == 0 and fencer_ip and fencer_ip != "" and is_ip_address(fencer_ip):
        fencer_ip = fencer_ip.strip().split("::")[-1]
        return test_ip_address(fencer_ip)

    has_zsha2, o = getstatusoutput("/usr/local/bin/zsha2 show-config")
    if has_zsha2 == 0:
        r, fencer_ip = getstatusoutput("/usr/local/bin/zsha2 show-config | grep -w 'gw' | awk '{print $NF}'")
        fencer_ip = fencer_ip.strip().strip("\",") if fencer_ip is not None else fencer_ip
        if r == 0 and is_ip_address(fencer_ip):
            return test_ip_address(fencer_ip)

    r, default_gateway = getstatusoutput("ip r get 8.8.8.8 | head -n1 | grep -o 'via.*dev' | awk '{print $2}'")
    if r == 0 and default_gateway and default_gateway != "" and is_ip_address(default_gateway):
        return test_ip_address(default_gateway)

    mgmt_ip = getoutput("timeout 3 vgs %s -otags --nolocking --noheading | tr ',' '\\n' | grep %s" % (vg_name, MANAGEMENT_TAG))
    mgmt_ip = mgmt_ip.strip().split("::")[-1]
    if not is_ip_address(mgmt_ip):
        logger.error("can not get mgmt nic")
        return False
    mgmt_device = getoutput("ip a | grep %s | awk '{print $NF}'" % mgmt_ip)
    return test_device(mgmt_device, 12) is None

def test_device(device, ttl=12):
    # type: (str, int) -> bool or None
    if ttl == 1:
        return None
    device = device.strip()
    is_bridge, o = getstatusoutput("brctl show %s" % device)
    if is_bridge == 0 and "can't get info" not in o.lower():
        _, o = getstatusoutput("brctl show %s | awk '{print $NF}' | grep -vw interfaces" % device)
        for i in o.splitlines():
            r = test_device(i.strip(), ttl-1)
            if r:
                return r

    if "." in device:
        return test_device(device.split(".", ttl-1)[0])

    is_bonding, o = getstatusoutput("cat /sys/class/net/%s/bonding/mii_status" % device)
    if is_bonding == 0:
        return "up" in o

    physical_nics = getoutput("find /sys/class/net -type l -not -lname '*virtual*' -printf '%f\\n'").splitlines()
    if True in [p.strip() == device for p in physical_nics]:
        return getoutput("/sys/class/net/%s/carrier" % device) == 1

    return None

def is_ip_address(ip):
    if re.match(r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$", ip):
        return True
    return False

def test_ip_address(ip):
    # type: (str) -> bool
    device = getoutput("ip r get %s | grep -Eo 'dev.*src' | awk '{print $2}'" % ip)
    for i in range(3):
        r, o = getstatusoutput("timeout 2 arping -b -c 1 -w 1 -I %s %s" % (device, ip))
        if r == 0:
            return True
    return False

def outdate_peer(resource_name):
    logger.debug("run outdate_peer %s" % resource_name)
    r, o = getstatusoutput("timeout 5 ssh -i %s %s@%s 'drbdadm outdate %s'" % (MINI_FENCER_KEY, PEER_USERNAME, PEER_MGMT_ADDR, resource_name))
    global OUTDATE_PEER_RESULT
    OUTDATE_PEER_RESULT = r == 0

def fence_self(resource_name, drbd_path):
    qemu_pid = getoutput("lsof -c qemu-kvm | grep -w %s | awk '{print $2}'" % drbd_path)
    run("kill -9 %s" % qemu_pid)
    run("drbdadm resume-io %s" % resource_name)
    run("drbdadm secondary %s" % resource_name)
    run("drbdadm outdate %s" % resource_name)

def timeout_callback():
    logger.error("timeout for 15 seconds! resume resource %s IO" % sys.argv[1])
    exit(4)

@set_timeout(20, timeout_callback)
def main():
    resource_name = sys.argv[1]
    logger.debug("fencer fired by resource %s" % resource_name)
    resource_path = getoutput(
        "grep -E 'disk.*/dev/' /etc/drbd.d/%s.res -m1 | awk '{print $2}' | tr -d ';'" % resource_name)
    vg_name = resource_path.split("/")[2]
    drbd_path = "/dev/drbd%s" % getoutput("cat /etc/drbd.d/%s.res | grep minor -m1 | awk '{print $NF}'" % resource_name).strip(";\n")
    try:
        t1 = threading.Thread(target=test_fencer, args=(vg_name, resource_name))
        t2 = threading.Thread(target=outdate_peer, args=(resource_name,))
        t1.start()
        t2.start()

        for i in range(32):
            if FENCE_GW_RESULT is None or OUTDATE_PEER_RESULT is None:
                time.sleep(0.5)
                continue
            elif FENCE_GW_RESULT is False and OUTDATE_PEER_RESULT is False:
                logger.info("resouce %s fence result: fence self" % resource_name)
                fence_self(resource_name, drbd_path)
                exit(0)
            else:
                logger.info("resouce %s fence result: not fence" % resource_name)
                exit(4)
        logger.error("timeout for 15 seconds! resume resource %s IO" % sys.argv[1])
        exit(4)
    except Exception as e:
        logger.info("resouce %s fence error, not proceeding" % resource_name)
        logger.error(e)
        if FENCE_GW_RESULT is False and OUTDATE_PEER_RESULT is False:
            exit(0)
        else:
            exit(4)

if __name__ == '__main__':
    logging.basicConfig(filename='/var/log/zstack/mini-fencer.log', level=logging.DEBUG,
                        format='%(asctime)s %(process)d %(levelname)s %(funcName)s %(message)s')
    main()
<commit_msg>Enhance mini fencer<commit_after>#!/usr/bin/env python
import functools
import sys
import commands
import logging
import time
import re
import signal
import threading

FENCER_TAG = "zs::ministorage::fencer"
MANAGEMENT_TAG = "zs::ministorage::management"

MINI_FENCER_KEY = "/usr/local/zstack/mini_fencer.key"
PEER_USERNAME = ""
PEER_MGMT_ADDR = ""

FENCE_GW_RESULT = None
OUTDATE_PEER_RESULT = None

OVERALL_TIMEOUT = 50

logger = logging.getLogger(__name__)

def set_timeout(num, callback):
    def wrap(func):
        def handle(signum, frame):
            raise RuntimeError

        def to_do(*args, **kwargs):
            try:
                signal.signal(signal.SIGALRM, handle)
                signal.alarm(num)
                r = func(*args, **kwargs)
                signal.alarm(0)
                return r
            except RuntimeError as e:
                callback()

        return to_do

    return wrap

def retry(times=5, sleep_time=1):
    def wrap(f):
        @functools.wraps(f)
        def inner(*args, **kwargs):
            last_err = None
            for i in range(0, times):
                try:
                    return f(*args, **kwargs)
                except Exception as e:
                    last_err = e
                    logger.error(e)
                    time.sleep(sleep_time)
            logger.error("The task failed,  detail:\n%s" % last_err)
        return inner
    return wrap

def getstatusoutput(c):
    # type: (str) -> (int, str)
    start_time = time.time()
    r, o = commands.getstatusoutput(c)
    end_time = time.time()
    logger.debug("command:[%s], returnCode:[%s], output:[%s], spendTime:[%s] ms" % (c, r, o, (end_time - start_time) * 1000))
    return r, o

def getoutput(c):
    return getstatusoutput(c)[1]

@retry()
def run(c):
    r, o = getstatusoutput(c)
    if r != 0:
        raise Exception("run command[%s] failed[return code: %s, output: %s]" % (c, r, o))

def test_fencer(vg_name, resource_name):
    logger.debug("run test_fencer %s" % resource_name)
    global FENCE_GW_RESULT
    FENCE_GW_RESULT = do_test_fencer(vg_name)

def do_test_fencer(vg_name):
    # type: (str) -> bool
    r, fencer_ip = getstatusoutput("timeout 3 vgs %s -otags --nolocking --noheading | tr ',' '\\n' | grep %s" % (vg_name, FENCER_TAG))
    if r == 0 and fencer_ip and fencer_ip != "" and is_ip_address(fencer_ip):
        fencer_ip = fencer_ip.strip().split("::")[-1]
        return test_ip_address(fencer_ip)

    has_zsha2, o = getstatusoutput("/usr/local/bin/zsha2 show-config")
    if has_zsha2 == 0:
        r, fencer_ip = getstatusoutput("/usr/local/bin/zsha2 show-config | grep -w 'gw' | awk '{print $NF}'")
        fencer_ip = fencer_ip.strip().strip("\",") if fencer_ip is not None else fencer_ip
        if r == 0 and is_ip_address(fencer_ip):
            return test_ip_address(fencer_ip)

    r, default_gateway = getstatusoutput("ip r get 8.8.8.8 | head -n1 | grep -o 'via.*dev' | awk '{print $2}'")
    if r == 0 and default_gateway and default_gateway != "" and is_ip_address(default_gateway):
        return test_ip_address(default_gateway)

    mgmt_ip = getoutput("timeout 3 vgs %s -otags --nolocking --noheading | tr ',' '\\n' | grep %s" % (vg_name, MANAGEMENT_TAG))
    mgmt_ip = mgmt_ip.strip().split("::")[-1]
    if not is_ip_address(mgmt_ip):
        logger.error("can not get mgmt nic")
        return False
    mgmt_device = getoutput("ip a | grep %s | awk '{print $NF}'" % mgmt_ip)
    return test_device(mgmt_device, 12) is None

def test_device(device, ttl=12):
    # type: (str, int) -> bool or None
    if ttl == 1:
        return None
    device = device.strip()
    is_bridge, o = getstatusoutput("brctl show %s" % device)
    if is_bridge == 0 and "can't get info" not in o.lower():
        _, o = getstatusoutput("brctl show %s | awk '{print $NF}' | grep -vw interfaces" % device)
        for i in o.splitlines():
            r = test_device(i.strip(), ttl-1)
            if r:
                return r

    if "." in device:
        return test_device(device.split(".", ttl-1)[0])

    is_bonding, o = getstatusoutput("cat /sys/class/net/%s/bonding/mii_status" % device)
    if is_bonding == 0:
        return "up" in o

    physical_nics = getoutput("find /sys/class/net -type l -not -lname '*virtual*' -printf '%f\\n'").splitlines()
    if True in [p.strip() == device for p in physical_nics]:
        return getoutput("/sys/class/net/%s/carrier" % device) == 1

    return None

def is_ip_address(ip):
    if re.match(r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$", ip):
        return True
    return False

def test_ip_address(ip):
    # type: (str) -> bool
    device = getoutput("ip r get %s | grep -Eo 'dev.*src' | awk '{print $2}'" % ip)
    for i in range(3):
        r, o = getstatusoutput("timeout 2 arping -b -c 1 -w 1 -I %s %s" % (device, ip))
        if r == 0:
            return True
    return False

def outdate_peer(resource_name):
    logger.debug("run outdate_peer %s" % resource_name)
    r, o = getstatusoutput("timeout 5 ssh -i %s %s@%s 'drbdadm outdate %s'" % (MINI_FENCER_KEY, PEER_USERNAME, PEER_MGMT_ADDR, resource_name))
    global OUTDATE_PEER_RESULT
    OUTDATE_PEER_RESULT = r == 0

def fence_self(resource_name, drbd_path):
    r, qemu_pid = getstatusoutput("timeout 3 lsof -c qemu-kvm | grep -w %s" % drbd_path)
    if r == 0:
        qemu_pid = qemu_pid.split()[1]
        run("timeout 3 kill -9 %s" % qemu_pid)
    run("timeout 3 drbdadm resume-io %s" % resource_name)
    run("timeout 3 drbdadm secondary %s" % resource_name)
    run("timeout 3 drbdadm outdate %s" % resource_name)

def timeout_callback():
    logger.error("overall timeout for %s seconds!" % OVERALL_TIMEOUT)
    if FENCE_GW_RESULT is False and OUTDATE_PEER_RESULT is False:
        logger.error("keep resource %s io suspend" % sys.argv[1])
        exit(0)
    else:
        logger.error("leave resource %s io resume" % sys.argv[1])
        exit(4)

# @set_timeout(OVERALL_TIMEOUT, timeout_callback)
def main():
    resource_name = sys.argv[1]
    logger.debug("fencer fired by resource %s" % resource_name)
    resource_path = getoutput(
        "grep -E 'disk.*/dev/' /etc/drbd.d/%s.res -m1 | awk '{print $2}' | tr -d ';'" % resource_name)
    vg_name = resource_path.split("/")[2]
    drbd_path = "/dev/drbd%s" % getoutput("cat /etc/drbd.d/%s.res | grep minor -m1 | awk '{print $NF}'" % resource_name).strip(";\n")
    try:
        t1 = threading.Thread(target=test_fencer, args=(vg_name, resource_name))
        t2 = threading.Thread(target=outdate_peer, args=(resource_name,))
        t1.start()
        t2.start()

        for i in range(32):
            if FENCE_GW_RESULT is None or OUTDATE_PEER_RESULT is None:
                time.sleep(0.5)
                continue
            elif FENCE_GW_RESULT is False and OUTDATE_PEER_RESULT is False:
                logger.info("resource %s fence result: fence self" % resource_name)
                fence_self(resource_name, drbd_path)
                exit(0)
            else:
                logger.info("resource %s fence result: not fence" % resource_name)
                exit(4)
        logger.error("timeout for 15 seconds! resume resource %s IO" % sys.argv[1])
        exit(4)
    except Exception as e:
        logger.error("resouce %s fence get error" % resource_name)
        logger.error(e.message)
        if FENCE_GW_RESULT is False and OUTDATE_PEER_RESULT is False:
            logger.error("keep resource %s io suspend" % resource_name)
            exit(0)
        else:
            logger.error("leave resource %s io resume" % resource_name)
            exit(4)

if __name__ == '__main__':
    logging.basicConfig(filename='/var/log/zstack/mini-fencer.log', level=logging.DEBUG,
                        format='%(asctime)s %(process)d %(levelname)s %(funcName)s %(message)s')
    main()
<|endoftext|>


<commit_before>#########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from cloudify.mocks import MockCloudifyContext
import mock
import unittest

class TestSequenceFunctions(unittest.TestCase):

    def setUp(self):
        pass

    def test_shuffle(self):
        print 1

if __name__ == '__main__':
    unittest.main()
<commit_msg>delete test<commit_after>#########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from cloudify.mocks import MockCloudifyContext
import mock
import unittest

class TestSequenceFunctions(unittest.TestCase):

    def setUp(self):
        pass

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

MAJOR_VERSION = '0'
MINOR_VERSION = '7'
PATCH_VERSION = '2'
VERSION = '.'.join((MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION,))
<commit_msg>change version<commit_after># -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

MAJOR_VERSION = '0'
MINOR_VERSION = '7'
PATCH_VERSION = '4'
VERSION = '.'.join((MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION,))
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

setup(
    name='django-debug-toolbar',
    version='0.10.0',
    description='A configurable set of panels that display various debug '
                'information about the current request/response.',
    long_description=open('README.rst').read(),
    author='Rob Hudson',
    author_email='rob@cogit8.org',
    url='https://github.com/django-debug-toolbar/django-debug-toolbar',
    download_url='https://github.com/django-debug-toolbar/django-debug-toolbar/downloads',
    license='BSD',
    packages=find_packages(exclude=('tests', 'example')),
    install_requires=[
        'django>=1.4.2,<1.6',
        'sqlparse',
    ],
    test_suite='runtests.runtests',
    include_package_data=True,
    zip_safe=False,                 # because we're including static files
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
)
<commit_msg>GitHub has removed the downloads section.<commit_after>from setuptools import setup, find_packages

setup(
    name='django-debug-toolbar',
    version='0.10.0',
    description='A configurable set of panels that display various debug '
                'information about the current request/response.',
    long_description=open('README.rst').read(),
    author='Rob Hudson',
    author_email='rob@cogit8.org',
    url='https://github.com/django-debug-toolbar/django-debug-toolbar',
    download_url='https://pypi.python.org/pypi/django-debug-toolbar',
    license='BSD',
    packages=find_packages(exclude=('tests', 'example')),
    install_requires=[
        'django>=1.4.2,<1.6',
        'sqlparse',
    ],
    test_suite='runtests.runtests',
    include_package_data=True,
    zip_safe=False,                 # because we're including static files
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
)
<|endoftext|>


<commit_before>#!/usr/bin/python
import json
import sys
import time
import random
import string
import os
import subprocess
import argparse

## random string generator
def randomword(length):
  return ''.join(random.choice(string.lowercase+string.uppercase+string.digits) for i in range(length))

def create_jobid ():
  return randomword(12)  # date+random_string

def get_start_time ():
  return time.strftime("%Y%m%d-%H:%M:%S-%Z")

def create_json_filename (jobid,json_dir):
  return json_dir + '/' + jobid + '.run.json'

## run command and check the output
## return value is [True/False, output_string(stdout)]
## If the command failed, the first value will be False and the output string will be null.
def run_command_out_check (command):
  with open(os.devnull,'w') as shutup:
    try:
      res=subprocess.check_output(command.split(" "), stderr=shutup)
      return([True,res])
    except subprocess.CalledProcessError:
      return([False,''])

def read_config( CONFIG_FILE, CONFIG_KEYS):

  ## 1. read .workflow.config.json file and get some variables
  with open( CONFIG_FILE ,'r') as f:
    cfg=json.load(f)

  # checking all the necessary keys exist
  for k in CONFIG_KEYS:
    if not cfg.has_key(k):
      sys.exit("The config file doesn't have key {}".format(k))

  return cfg

def create_json( a, json_dir, jobid, copy_to_s3 ):  # a is the final_args dictionary. json_dir is the output directory for the json file

  ## create jobid here
  if not jobid:
    jobid = create_jobid()

  ## start time
  start_time = get_start_time()

  ##  pre is a dictionary to be printed as a pre-run json file.
  pre = {  'JOBID': jobid,
           'App': {
              'App_name': a['app_name'],
              'App_version': a['app_version'],
              'cwl_url': a['cwl_directory'],
              'main_cwl': a['cwl'],
              'other_cwl_files': a['cwl_children']
           },
           'Input': {
              'Input_files_data': {},  ## fill in later (below)
              'Input_files_reference': {},   ## fill in later (below)
              'Input_parameters': a['input_parameters']
           },
           'Output': {
              'output_bucket_directory': a['output_bucket_directory']
           },
           'Instance_type': a['instance_type'],
           'EBS_SIZE': a['storage_size'],
           'EBS_TYPE': 'io1',
           'EBS_IOPS': a['storage_iops'],
           "AMI_ID": "ami-78c13615",
           "start_time" : start_time
        }

  # fill in input_files and input_reference_files (restructured)
  for item,value in a['input_files'].iteritems():
    pre['Input']['Input_files_data'][item]={'class':'File','dir':a['input_files_directory'],'path':value}
  for item,value in a['input_reference_files'].iteritems():
    pre['Input']['Input_files_reference'][item]={'class':'File','dir':a['input_reference_files_directory'],'path':value}

  # wrap
  pre={ 'Job': pre }

  ## writing to a json file
  json_filename = create_json_filename(jobid, json_dir)
  try:
    os.stat(json_dir)
  except:
    os.makedirs(json_dir)

  ## write to new json file
  with open(json_filename,'w') as json_new_f:
    json.dump(pre,json_new_f,indent=4,sort_keys=True)

  # copy the json file to the s3 bucket
  if copy_to_s3==True:
    command = "aws s3 cp ./{json_dir}/{jobid}.run.json s3://{json_bucket}/{jobid}.run.json".format(json_bucket=json_bucket, jobid=jobid, json_dir=json_dir)
    run_command_out_check (command)

  ## print & retur JOBID
  print(jobid)
  return(jobid)

def launch_instance (par, jobid):

  ## Create a userdata script to pass to the instance. The userdata script is run_workflow.$JOBID.sh.
  command="./create_run_workflow.sh {jobid} {dir}".format(jobid=jobid,dir=par['userdata_dir'])
  run_command_out_check (command)

  ## launch an instance
  print "launching an instance..."

  Userdata_file = "run_workflow.{jobid}.sh".format(jobid=jobid)

  launch_command = "aws ec2 run-instances --image-id {ami} --instance-type {instance_type} --instance-initiated-shutdown-behavior terminate --count 1 --enable-api-termination  --block-device-mappings DeviceName=/dev/sdb,Ebs=\"{{VolumeSize={EBS_SIZE},VolumeType={EBS_TYPE},Iops={EBS_IOPS},DeleteOnTermination=true}}\" --iam-instance-profile Arn={arn} --ebs-optimized --user-data={userdata}".format(ami=par['worker_ami_id'], instance_type=par['instance_type'], arn=par['s3_access_arn'], EBS_SIZE=par['storage_size'], EBS_TYPE='io1', EBS_IOPS=par['storage_iops'], userdata='file://'+Userdata_file)
  if par['keyname'] != '':
    launch_command += " --key-name {keyname}".format(keyname=par['keyname'])
  instance_launch_logstr=subprocess.check_output(launch_command, shell=True) # capturing stdout from the launch command
  instance_launch_log=json.loads(instance_launch_logstr)
  instance_id = instance_launch_log['Instances'][0]['InstanceId']

  # get public IP for the instance (This may not happen immediately)
  instance_desc_command = "aws ec2 describe-instances --instance-id={instance_id}".format(instance_id=instance_id)
  instance_desc_logstr=[False,'']
  while instance_desc_logstr[0]==False:  ## keep trying until you get the result.
    instance_desc_logstr= run_command_out_check (instance_desc_command)
  instance_desc_log=json.loads(instance_desc_logstr[1])
  instance_ip = instance_desc_log['Reservations'][0]['Instances'][0]['PublicIpAddress']

  ## 5. Add to the job list
  with open(par['job_list_file'],'a') as fo:
    fo.write("{}\t{}\t{}\t{}\t{}\n".format(jobid, instance_id, par['instance_type'], instance_ip, par['job_tag']))

## main
parser = argparse.ArgumentParser()

parser.add_argument("-c", "--cwl", help="main cwl file name")
parser.add_argument("-cd", "--cwl_directory", help="the url and subdirectories for the main cwl file (override config)")
parser.add_argument("-co", "--cwl_children", help="names of the other cwl files used by main cwl file, delimiated by comma")
parser.add_argument("-a", "--app_name", help="name of the app")
parser.add_argument("-av", "--app_version", help="version of the app")
parser.add_argument("-i", "--input_files", help="input files in json format (parametername:filename)")
parser.add_argument("-ir", "--input_reference_files", help="input reference files in json format (parametername:filename)")
parser.add_argument("-ip", "--input_parameters", help="input parameters in json format (parametername:value)")
parser.add_argument("-id", "--input_files_directory", help="bucket name and subdirectory for input files")
parser.add_argument("-ird", "--input_reference_files_directory", help="bucket name and subdirectory for input reference files (override config)")
parser.add_argument("-o", "--output_bucket_directory", help="bucket name and subdirectory for output files and logs (override config)")
parser.add_argument("-t", "--instance_type", help="EC2 instance type (default set in config)")
parser.add_argument("-s", "--storage_size", help="EBS storage size in GB (default set in config)")
parser.add_argument("-IO", "--storage_iops", help="EBS storage IOPS (default set in config)")
parser.add_argument("-jd", "--json_dir", help="Local directory in which the output json file will be written (default set in config)")
parser.add_argument("-J", "--job_id", help="Manually assign job ID as specififed (default: randomly generated)")
parser.add_argument("-u", "--copy_to_s3", help="Upload or copy the json file to S3 bucket json_bucket", action="store_true")
parser.add_argument("-e", "--launch_instance", help="Launch instance based on the json file.", action="store_true")

args = parser.parse_args()

## default variables
CONFIG_FILE=".tibanna.config"
CONFIG_KEYS=["reference_S3_bucket","output_S3_bucket","s3_access_arn","keyname","worker_ami_id","default_instance_type","default_ebs_size","ebs_iops","userdata_dir","json_dir","json_bucket","cwl_url","job_list_file"]
cfg=read_config(CONFIG_FILE, CONFIG_KEYS)

## parameters that will go into the json file
final_args={
 'cwl': '',  ## required
 'cwl_directory': cfg['cwl_url'],
 'cwl_children': [],
 'app_name': '',
 'app_version': '',
 'input_files': {},
 'input_reference_files': {},
 'input_parameters': {},
 'input_files_directory': '', ## required if input_files is not null
 'input_reference_files_directory': cfg['reference_S3_bucket'],
 'output_bucket_directory': cfg['output_S3_bucket'],
 'instance_type': cfg['default_instance_type'],
 'storage_size': cfg['default_ebs_size'],
 'storage_iops': cfg['ebs_iops']
}
# local directory in which the json file will be first created.
json_dir=cfg['json_dir']
# bucket name to which the json file will be sent.
json_bucket=cfg['json_bucket']

# parameters needed to launch an instance
par={
 's3_access_arn': cfg['s3_access_arn'],
 'worker_ami_id': cfg['worker_ami_id'],
 'keyname': cfg['keyname'],
 'userdata_dir': cfg['userdata_dir'],
 'instance_type': cfg['default_instance_type'], # redundant with final_args
 'storage_size': cfg['default_ebs_size'], # redudant with final_args
 'storage_iops': cfg['ebs_iops'], # redundant with final_args
 'job_list_file': cfg['job_list_file'],
 'job_tag': '' # app_name in final_args
}

if args.cwl:
  final_args['cwl']=args.cwl
else:
  sys.exit("cwl field is required")

if args.cwl_directory:
  final_args['cwl_directory']=args.cwl_directory

if args.cwl_children:
  final_args['cwl_children']=args.cwl_children.split(',')

if args.app_name:
  final_args['app_name']=args.app_name

if args.app_version:
  final_args['app_version']=args.app_version

if args.input_files:
  final_args['input_files']=json.loads(args.input_files)

if args.input_reference_files:
  final_args['input_reference_files']=json.loads(args.input_reference_files)

if args.input_parameters:
  final_args['input_parameters']=json.loads(args.input_parameters)

if args.input_files_directory:
  final_args['input_files_directory'] = args.input_files_directory
elif bool(final_args['input_files']):
  sys.exit("input_files_directory must be provided if input_files is provided.")

if args.input_reference_files_directory:
  final_args['input_reference_files_directory'] = args.input_reference_files_directory

if args.output_bucket_directory:
  final_args['output_bucket_directory']=args.output_bucket_directory

if args.instance_type:
  final_args['instance_type']=args.instance_type

if args.storage_size:
  final_args['storage_size']=int(args.storage_size)

if args.storage_iops:
  final_args['storage_iops']=int(args.storage_iops)

if args.json_dir:
  json_dir=args.json_dir

# make sure these parameters are consistent between par and a.
par['instance_type'] =final_args['instance_type']
par['storage_size']= final_args['storage_size']
par['storage_iops']= final_args['storage_iops']
par['job_tag']=final_args['app_name']

# create json and copy to s3
jobid=create_json(final_args, json_dir, args.job_id, args.copy_to_s3 )

# launch instance and execute workflow
if args.launch_instance:
  launch_instance(par, jobid)

<commit_msg>fixed userdata directory recognition<commit_after>#!/usr/bin/python
import json
import sys
import time
import random
import string
import os
import subprocess
import argparse

## random string generator
def randomword(length):
  return ''.join(random.choice(string.lowercase+string.uppercase+string.digits) for i in range(length))

def create_jobid ():
  return randomword(12)  # date+random_string

def get_start_time ():
  return time.strftime("%Y%m%d-%H:%M:%S-%Z")

def create_json_filename (jobid,json_dir):
  return json_dir + '/' + jobid + '.run.json'

## run command and check the output
## return value is [True/False, output_string(stdout)]
## If the command failed, the first value will be False and the output string will be null.
def run_command_out_check (command):
  with open(os.devnull,'w') as shutup:
    try:
      res=subprocess.check_output(command.split(" "), stderr=shutup)
      return([True,res])
    except subprocess.CalledProcessError:
      return([False,''])

def read_config( CONFIG_FILE, CONFIG_KEYS):

  ## 1. read .workflow.config.json file and get some variables
  with open( CONFIG_FILE ,'r') as f:
    cfg=json.load(f)

  # checking all the necessary keys exist
  for k in CONFIG_KEYS:
    if not cfg.has_key(k):
      sys.exit("The config file doesn't have key {}".format(k))

  return cfg

def create_json( a, json_dir, jobid, copy_to_s3 ):  # a is the final_args dictionary. json_dir is the output directory for the json file

  ## create jobid here
  if not jobid:
    jobid = create_jobid()

  ## start time
  start_time = get_start_time()

  ##  pre is a dictionary to be printed as a pre-run json file.
  pre = {  'JOBID': jobid,
           'App': {
              'App_name': a['app_name'],
              'App_version': a['app_version'],
              'cwl_url': a['cwl_directory'],
              'main_cwl': a['cwl'],
              'other_cwl_files': a['cwl_children']
           },
           'Input': {
              'Input_files_data': {},  ## fill in later (below)
              'Input_files_reference': {},   ## fill in later (below)
              'Input_parameters': a['input_parameters']
           },
           'Output': {
              'output_bucket_directory': a['output_bucket_directory']
           },
           'Instance_type': a['instance_type'],
           'EBS_SIZE': a['storage_size'],
           'EBS_TYPE': 'io1',
           'EBS_IOPS': a['storage_iops'],
           "AMI_ID": "ami-78c13615",
           "start_time" : start_time
        }

  # fill in input_files and input_reference_files (restructured)
  for item,value in a['input_files'].iteritems():
    pre['Input']['Input_files_data'][item]={'class':'File','dir':a['input_files_directory'],'path':value}
  for item,value in a['input_reference_files'].iteritems():
    pre['Input']['Input_files_reference'][item]={'class':'File','dir':a['input_reference_files_directory'],'path':value}

  # wrap
  pre={ 'Job': pre }

  ## writing to a json file
  json_filename = create_json_filename(jobid, json_dir)
  try:
    os.stat(json_dir)
  except:
    os.makedirs(json_dir)

  ## write to new json file
  with open(json_filename,'w') as json_new_f:
    json.dump(pre,json_new_f,indent=4,sort_keys=True)

  # copy the json file to the s3 bucket
  if copy_to_s3==True:
    command = "aws s3 cp ./{json_dir}/{jobid}.run.json s3://{json_bucket}/{jobid}.run.json".format(json_bucket=json_bucket, jobid=jobid, json_dir=json_dir)
    run_command_out_check (command)

  ## print & retur JOBID
  print(jobid)
  return(jobid)

def launch_instance (par, jobid):

  ## Create a userdata script to pass to the instance. The userdata script is run_workflow.$JOBID.sh.
  command="./create_run_workflow.sh {jobid} {dir}".format(jobid=jobid,dir=par['userdata_dir'])
  run_command_out_check (command)

  ## launch an instance
  print "launching an instance..."

  Userdata_file = "{dir}/run_workflow.{jobid}.sh".format(jobid=jobid,dir=par['userdata_dir'])

  launch_command = "aws ec2 run-instances --image-id {ami} --instance-type {instance_type} --instance-initiated-shutdown-behavior terminate --count 1 --enable-api-termination  --block-device-mappings DeviceName=/dev/sdb,Ebs=\"{{VolumeSize={EBS_SIZE},VolumeType={EBS_TYPE},Iops={EBS_IOPS},DeleteOnTermination=true}}\" --iam-instance-profile Arn={arn} --ebs-optimized --user-data={userdata}".format(ami=par['worker_ami_id'], instance_type=par['instance_type'], arn=par['s3_access_arn'], EBS_SIZE=par['storage_size'], EBS_TYPE='io1', EBS_IOPS=par['storage_iops'], userdata='file://'+Userdata_file)
  if par['keyname'] != '':
    launch_command += " --key-name {keyname}".format(keyname=par['keyname'])
  instance_launch_logstr=subprocess.check_output(launch_command, shell=True) # capturing stdout from the launch command
  instance_launch_log=json.loads(instance_launch_logstr)
  instance_id = instance_launch_log['Instances'][0]['InstanceId']

  # get public IP for the instance (This may not happen immediately)
  instance_desc_command = "aws ec2 describe-instances --instance-id={instance_id}".format(instance_id=instance_id)
  instance_desc_logstr=[False,'']
  while instance_desc_logstr[0]==False:  ## keep trying until you get the result.
    instance_desc_logstr= run_command_out_check (instance_desc_command)
  instance_desc_log=json.loads(instance_desc_logstr[1])
  instance_ip = instance_desc_log['Reservations'][0]['Instances'][0]['PublicIpAddress']

  ## 5. Add to the job list
  with open(par['job_list_file'],'a') as fo:
    fo.write("{}\t{}\t{}\t{}\t{}\n".format(jobid, instance_id, par['instance_type'], instance_ip, par['job_tag']))

## main
parser = argparse.ArgumentParser()

parser.add_argument("-c", "--cwl", help="main cwl file name")
parser.add_argument("-cd", "--cwl_directory", help="the url and subdirectories for the main cwl file (override config)")
parser.add_argument("-co", "--cwl_children", help="names of the other cwl files used by main cwl file, delimiated by comma")
parser.add_argument("-a", "--app_name", help="name of the app")
parser.add_argument("-av", "--app_version", help="version of the app")
parser.add_argument("-i", "--input_files", help="input files in json format (parametername:filename)")
parser.add_argument("-ir", "--input_reference_files", help="input reference files in json format (parametername:filename)")
parser.add_argument("-ip", "--input_parameters", help="input parameters in json format (parametername:value)")
parser.add_argument("-id", "--input_files_directory", help="bucket name and subdirectory for input files")
parser.add_argument("-ird", "--input_reference_files_directory", help="bucket name and subdirectory for input reference files (override config)")
parser.add_argument("-o", "--output_bucket_directory", help="bucket name and subdirectory for output files and logs (override config)")
parser.add_argument("-t", "--instance_type", help="EC2 instance type (default set in config)")
parser.add_argument("-s", "--storage_size", help="EBS storage size in GB (default set in config)")
parser.add_argument("-IO", "--storage_iops", help="EBS storage IOPS (default set in config)")
parser.add_argument("-jd", "--json_dir", help="Local directory in which the output json file will be written (default set in config)")
parser.add_argument("-J", "--job_id", help="Manually assign job ID as specififed (default: randomly generated)")
parser.add_argument("-u", "--copy_to_s3", help="Upload or copy the json file to S3 bucket json_bucket", action="store_true")
parser.add_argument("-e", "--launch_instance", help="Launch instance based on the json file.", action="store_true")

args = parser.parse_args()

## default variables
CONFIG_FILE=".tibanna.config"
CONFIG_KEYS=["reference_S3_bucket","output_S3_bucket","s3_access_arn","keyname","worker_ami_id","default_instance_type","default_ebs_size","ebs_iops","userdata_dir","json_dir","json_bucket","cwl_url","job_list_file"]
cfg=read_config(CONFIG_FILE, CONFIG_KEYS)

## parameters that will go into the json file
final_args={
 'cwl': '',  ## required
 'cwl_directory': cfg['cwl_url'],
 'cwl_children': [],
 'app_name': '',
 'app_version': '',
 'input_files': {},
 'input_reference_files': {},
 'input_parameters': {},
 'input_files_directory': '', ## required if input_files is not null
 'input_reference_files_directory': cfg['reference_S3_bucket'],
 'output_bucket_directory': cfg['output_S3_bucket'],
 'instance_type': cfg['default_instance_type'],
 'storage_size': cfg['default_ebs_size'],
 'storage_iops': cfg['ebs_iops']
}
# local directory in which the json file will be first created.
json_dir=cfg['json_dir']
# bucket name to which the json file will be sent.
json_bucket=cfg['json_bucket']

# parameters needed to launch an instance
par={
 's3_access_arn': cfg['s3_access_arn'],
 'worker_ami_id': cfg['worker_ami_id'],
 'keyname': cfg['keyname'],
 'userdata_dir': cfg['userdata_dir'],
 'instance_type': cfg['default_instance_type'], # redundant with final_args
 'storage_size': cfg['default_ebs_size'], # redudant with final_args
 'storage_iops': cfg['ebs_iops'], # redundant with final_args
 'job_list_file': cfg['job_list_file'],
 'job_tag': '' # app_name in final_args
}

if args.cwl:
  final_args['cwl']=args.cwl
else:
  sys.exit("cwl field is required")

if args.cwl_directory:
  final_args['cwl_directory']=args.cwl_directory

if args.cwl_children:
  final_args['cwl_children']=args.cwl_children.split(',')

if args.app_name:
  final_args['app_name']=args.app_name

if args.app_version:
  final_args['app_version']=args.app_version

if args.input_files:
  final_args['input_files']=json.loads(args.input_files)

if args.input_reference_files:
  final_args['input_reference_files']=json.loads(args.input_reference_files)

if args.input_parameters:
  final_args['input_parameters']=json.loads(args.input_parameters)

if args.input_files_directory:
  final_args['input_files_directory'] = args.input_files_directory
elif bool(final_args['input_files']):
  sys.exit("input_files_directory must be provided if input_files is provided.")

if args.input_reference_files_directory:
  final_args['input_reference_files_directory'] = args.input_reference_files_directory

if args.output_bucket_directory:
  final_args['output_bucket_directory']=args.output_bucket_directory

if args.instance_type:
  final_args['instance_type']=args.instance_type

if args.storage_size:
  final_args['storage_size']=int(args.storage_size)

if args.storage_iops:
  final_args['storage_iops']=int(args.storage_iops)

if args.json_dir:
  json_dir=args.json_dir

# make sure these parameters are consistent between par and a.
par['instance_type'] =final_args['instance_type']
par['storage_size']= final_args['storage_size']
par['storage_iops']= final_args['storage_iops']
par['job_tag']=final_args['app_name']

# create json and copy to s3
jobid=create_json(final_args, json_dir, args.job_id, args.copy_to_s3 )

# launch instance and execute workflow
if args.launch_instance:
  launch_instance(par, jobid)

<|endoftext|>


<commit_before>import plugins, asyncio

def _initialise(bot):
   plugins.register_user_command(["botdoc"])
   plugins.register_admin_command(["setbotdoc"])

@asyncio.coroutine
def botdoc(bot, event, *args):
   yield from bot.coro_send_message(event.conv,'https://goo.gl/xxxx')

@asyncio.coroutine
def setbotdoc(bot, event, *args):
   yield from bot.coro_send_message(event.conv,'https://goo.gl/xxxx')
<commit_msg>Update botdoc.py<commit_after>import asyncio, re, logging, json, random
import hangups
import plugins

logger = logging.getLogger(__name__)

def _initialise(bot):
   plugins.register_user_command(["botdoc"])
   plugins.register_admin_command(["setbotdoc"])

@asyncio.coroutine
def botdoc(bot, event, *args):
   """Shows the bot related documentation"""
   text = bot.user_memory_get('botdocmemory', 'botdoc')
   if text is None:
      message = u'There is no documentation for this HO bot! :(' % (event.user.full_name, name)
   else
      message = u'%s' % (text)
   yield from bot.coro_send_message(event.conv,message)

@asyncio.coroutine
def setbotdoc(bot, event, *args):
   """Set the botdoc message. Only admins can set the botdoc message. With /bot setbotdoc clear you can clear the botdoc message."""
   parameters = list(args)
   if parameters[0] == "clear":
      bot.user_memory_set('botdocmemory', 'botdoc', None)
   else:
      bot.user_memory_set('botdocmemory', 'botdoc', ' '.join(args))
      text = bot.user_memory_get('botdocmemory', 'botdoc')
      message = u'Botdoc message stored: %s' % (text)
      yield from bot.coro_send_message(event.conv,message)
<|endoftext|>


<commit_before>"""
This file is part of VoltDB.

Copyright (C) 2008-2016 VoltDB Inc.

This file contains original code and/or modifications of original code.
Any modifications made by VoltDB Inc. are licensed under the following
terms and conditions:

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
"""

import unittest
import requests
import xmlrunner
import socket
import time
import json

__host_name__ = socket.gethostname()
__host_or_ip__ = socket.gethostbyname(__host_name__)

URL = 'http://%s:8000/api/1.0/databases/1/servers/' % \
(__host_or_ip__)
__db_url__ = 'http://%s:8000/api/1.0/databases/' % \
             (__host_or_ip__)

class Server(unittest.TestCase):
    """
    Test case for adding and deleting servers
    """
    def setUp(self):
        """
        # Create a db
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        db_data = {'name': 'testDB'}
        response = requests.post(__db_url__, json=db_data, headers=headers)
        if response.status_code == 201:
            self.assertEqual(response.status_code, 201)
        else:
            self.assertEqual(response.status_code, 404)
        # Create a server
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)

            data = {'description': 'test', 'hostname': __host_or_ip__, 'name': 'test'}
            response = requests.post(url, json=data, headers=headers)
            if response.status_code == 201:
                self.assertEqual(response.status_code, 201)
            else:
                self.assertEqual(response.status_code, 404)
        else:
            print "The database list is empty"

    def tearDown(self):
        """
        Delete the server
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']
            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)
            response = requests.get(url)
            value = response.json()
            if value:
                server_length = len(value['members'])
                last_server_id = value['members'][server_length-1]['id']
                print "ServerId to be deleted is " + str(last_server_id)
                url += str(last_server_id)
                response = requests.delete(url)
                self.assertEqual(response.status_code, 200)
                # Delete database
                db_url = __db_url__ + str(last_db_id)
                response = requests.delete(db_url)
                self.assertEqual(response.status_code, 200)
            else:
                print "The Server list is empty"
        else:
            print "The database list is empty"

class Cluster(unittest.TestCase):
    """
    Test case for adding and deleting servers with ports other than the default ports
    """
    def setUp(self):
        """
        # Create a db
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        db_data = {'name': 'testDB1'}
        response = requests.post(__db_url__, json=db_data, headers=headers)
        if response.status_code == 201:
            self.assertEqual(response.status_code, 201)
        else:
            self.assertEqual(response.status_code, 404)
        # Create a server
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)

            data = {'description': 'test', 'hostname': __host_or_ip__, 'name': 'test', 'client-listener': '21214', 'admin-listener': '12345'}
            response = requests.post(url, json=data, headers=headers)
            if response.status_code == 201:
                self.assertEqual(response.status_code, 201)
            else:
                self.assertEqual(response.status_code, 404)
        else:
            print "The database list is empty"

    def tearDown(self):
        """
        Delete the server
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']
            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)
            response = requests.get(url)
            value = response.json()
            if value:
                server_length = len(value['members'])
                last_server_id = value['members'][server_length-1]['id']
                print "ServerId to be deleted is " + str(last_server_id)
                url += str(last_server_id)
                response = requests.delete(url)
                self.assertEqual(response.status_code, 200)
                # Delete database
                db_url = __db_url__ + str(last_db_id)
                response = requests.delete(db_url)
                self.assertEqual(response.status_code, 200)
            else:
                print "The Server list is empty"
        else:
            print "The database list is empty"

class DefaultStartServer(Server):
    """
    Create Server
    """

    def test_start_stop_server(self):
        """
        ensure Start and stop server is working properly
        """
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/start' % \
                (__host_or_ip__,last_db_id)
            print "Starting..."
            response = requests.put(url)
            value = response.json()
            if not value['statusstring']:
                print "The Server list is empty"
            elif "Start request sent successfully to servers" in value['statusstring']:
                self.assertEqual(response.status_code, 200)
                time.sleep(20)
                CheckServerStatus(self, last_db_id, 'running')
                time.sleep(10)
                print "Stopping Cluster...."
                url_stop = 'http://%s:8000/api/1.0/databases/%u/stop' % \
                (__host_or_ip__,last_db_id)
                response = requests.put(url_stop)
                value = response.json()
                if "Server shutdown successfully." in value['statusstring']:
                    self.assertEqual(response.status_code, 200)
                    time.sleep(15)
                    CheckServerStatus(self, last_db_id, 'stopped')
            elif response.status_code == 500:
                self.assertEqual(response.status_code, 500)

class StartServer(Cluster):
    """
    Start Server in different port
    """

    def test_start_stop_server_with_different_ports(self):
        """
        ensure Start and stop server is working properly using ports other than default ports
        """
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/start' % \
                (__host_or_ip__,last_db_id)

            response = requests.put(url)
            print "Starting...."
            value = response.json()
            if not value['statusstring']:
                print "error"
            elif "Start request sent successfully to servers" in value['statusstring']:
                self.assertEqual(response.status_code, 200)
                time.sleep(20)
                CheckServerStatus(self, last_db_id, 'running')
                time.sleep(10)
                print "Stopping...."
                url_stop = 'http://%s:8000/api/1.0/databases/%u/stop' % \
                (__host_or_ip__,last_db_id)
                response = requests.put(url_stop)
                value = response.json()
                if "Server shutdown successfully." in value['statusstring']:
                    self.assertEqual(response.status_code, 200)
                    time.sleep(15)
                    CheckServerStatus(self, last_db_id, 'stopped')
            elif response.status_code == 500:
                self.assertEqual(response.status_code, 500)

class DefaultRecoverServer(Server):
    """
    Create Server
    """

    def test_recover_stop_server(self):
        """
        ensure Start and stop server is working properly
        """
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/recover' % \
                (__host_or_ip__,last_db_id)
            print "Recovering..."
            response = requests.put(url)
            value = response.json()

            if not value['statusstring']:
                print "Error"
            elif "FATAL: VoltDB Community Edition" in value['statusstring']:
                print "Voltdb recover is only supported in Enterprise Edition"
            elif "Start request sent successfully to servers" in value['statusstring']:
                self.assertEqual(response.status_code, 200)
                time.sleep(20)
                CheckServerStatus(self, last_db_id, 'running')
                time.sleep(10)
                print "Stopping Cluster...."
                url_stop = 'http://%s:8000/api/1.0/databases/%u/stop' % \
                (__host_or_ip__,last_db_id)
                response = requests.put(url_stop)
                value = response.json()
                if "Connection broken" in value['statusstring']:
                    self.assertEqual(response.status_code, 200)
                    time.sleep(15)
                    CheckServerStatus(self, last_db_id, 'stopped')
            elif response.status_code == 500:
                self.assertEqual(response.status_code, 500)

def CheckServerStatus(self, last_db_id, status):
    status_url = 'http://%s:8000/api/1.0/databases/%u/status/' % \
    (__host_or_ip__,last_db_id)
    print "Checking status..."
    response = requests.get(status_url)
    value = response.json()
    if value['status'] and value['status'][0]['status']:
        print "Status: " + value['status'][0]['status']
        self.assertEqual(value['status'][0]['status'], status)
        self.assertEqual(value['serverDetails'][0][__host_or_ip__]['status'], status)
    else:
        assert False

if __name__ == '__main__':
    unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports'))
    unittest.main()
<commit_msg>VDM-71 Modified the status string in test case.<commit_after>"""
This file is part of VoltDB.

Copyright (C) 2008-2016 VoltDB Inc.

This file contains original code and/or modifications of original code.
Any modifications made by VoltDB Inc. are licensed under the following
terms and conditions:

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
"""

import unittest
import requests
import xmlrunner
import socket
import time
import json

__host_name__ = socket.gethostname()
__host_or_ip__ = socket.gethostbyname(__host_name__)

URL = 'http://%s:8000/api/1.0/databases/1/servers/' % \
(__host_or_ip__)
__db_url__ = 'http://%s:8000/api/1.0/databases/' % \
             (__host_or_ip__)

class Server(unittest.TestCase):
    """
    Test case for adding and deleting servers
    """
    def setUp(self):
        """
        # Create a db
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        db_data = {'name': 'testDB'}
        response = requests.post(__db_url__, json=db_data, headers=headers)
        if response.status_code == 201:
            self.assertEqual(response.status_code, 201)
        else:
            self.assertEqual(response.status_code, 404)
        # Create a server
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)

            data = {'description': 'test', 'hostname': __host_or_ip__, 'name': 'test'}
            response = requests.post(url, json=data, headers=headers)
            if response.status_code == 201:
                self.assertEqual(response.status_code, 201)
            else:
                self.assertEqual(response.status_code, 404)
        else:
            print "The database list is empty"

    def tearDown(self):
        """
        Delete the server
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']
            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)
            response = requests.get(url)
            value = response.json()
            if value:
                server_length = len(value['members'])
                last_server_id = value['members'][server_length-1]['id']
                print "ServerId to be deleted is " + str(last_server_id)
                url += str(last_server_id)
                response = requests.delete(url)
                self.assertEqual(response.status_code, 200)
                # Delete database
                db_url = __db_url__ + str(last_db_id)
                response = requests.delete(db_url)
                self.assertEqual(response.status_code, 200)
            else:
                print "The Server list is empty"
        else:
            print "The database list is empty"

class Cluster(unittest.TestCase):
    """
    Test case for adding and deleting servers with ports other than the default ports
    """
    def setUp(self):
        """
        # Create a db
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        db_data = {'name': 'testDB1'}
        response = requests.post(__db_url__, json=db_data, headers=headers)
        if response.status_code == 201:
            self.assertEqual(response.status_code, 201)
        else:
            self.assertEqual(response.status_code, 404)
        # Create a server
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)

            data = {'description': 'test', 'hostname': __host_or_ip__, 'name': 'test', 'client-listener': '21214', 'admin-listener': '12345'}
            response = requests.post(url, json=data, headers=headers)
            if response.status_code == 201:
                self.assertEqual(response.status_code, 201)
            else:
                self.assertEqual(response.status_code, 404)
        else:
            print "The database list is empty"

    def tearDown(self):
        """
        Delete the server
        """
        headers = {'Content-Type': 'application/json; charset=utf-8'}
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']
            url = 'http://%s:8000/api/1.0/databases/%u/servers/' % \
                (__host_or_ip__,last_db_id)
            response = requests.get(url)
            value = response.json()
            if value:
                server_length = len(value['members'])
                last_server_id = value['members'][server_length-1]['id']
                print "ServerId to be deleted is " + str(last_server_id)
                url += str(last_server_id)
                response = requests.delete(url)
                self.assertEqual(response.status_code, 200)
                # Delete database
                db_url = __db_url__ + str(last_db_id)
                response = requests.delete(db_url)
                self.assertEqual(response.status_code, 200)
            else:
                print "The Server list is empty"
        else:
            print "The database list is empty"

class DefaultStartServer(Server):
    """
    Create Server
    """

    def test_start_stop_server(self):
        """
        ensure Start and stop server is working properly
        """
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/start' % \
                (__host_or_ip__,last_db_id)
            print "Starting..."
            response = requests.put(url)
            value = response.json()
            if not value['statusstring']:
                print "The Server list is empty"
            elif "Start request sent successfully to servers" in value['statusstring']:
                self.assertEqual(response.status_code, 200)
                time.sleep(20)
                CheckServerStatus(self, last_db_id, 'running')
                time.sleep(10)
                print "Stopping Cluster...."
                url_stop = 'http://%s:8000/api/1.0/databases/%u/stop' % \
                (__host_or_ip__,last_db_id)
                response = requests.put(url_stop)
                value = response.json()
                if "Server shutdown successfully." in value['statusstring']:
                    self.assertEqual(response.status_code, 200)
                    time.sleep(15)
                    CheckServerStatus(self, last_db_id, 'stopped')
            elif response.status_code == 500:
                self.assertEqual(response.status_code, 500)

class StartServer(Cluster):
    """
    Start Server in different port
    """

    def test_start_stop_server_with_different_ports(self):
        """
        ensure Start and stop server is working properly using ports other than default ports
        """
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/start' % \
                (__host_or_ip__,last_db_id)

            response = requests.put(url)
            print "Starting...."
            value = response.json()
            if not value['statusstring']:
                print "error"
            elif "Start request sent successfully to servers" in value['statusstring']:
                self.assertEqual(response.status_code, 200)
                time.sleep(20)
                CheckServerStatus(self, last_db_id, 'running')
                time.sleep(10)
                print "Stopping...."
                url_stop = 'http://%s:8000/api/1.0/databases/%u/stop' % \
                (__host_or_ip__,last_db_id)
                response = requests.put(url_stop)
                value = response.json()
                if "Server shutdown successfully." in value['statusstring']:
                    self.assertEqual(response.status_code, 200)
                    time.sleep(15)
                    CheckServerStatus(self, last_db_id, 'stopped')
            elif response.status_code == 500:
                self.assertEqual(response.status_code, 500)

class DefaultRecoverServer(Server):
    """
    Create Server
    """

    def test_recover_stop_server(self):
        """
        ensure Start and stop server is working properly
        """
        response = requests.get(__db_url__)
        value = response.json()
        if value:
            db_length = len(value['databases'])
            last_db_id = value['databases'][db_length-1]['id']

            url = 'http://%s:8000/api/1.0/databases/%u/recover' % \
                (__host_or_ip__,last_db_id)
            print "Recovering..."
            response = requests.put(url)
            value = response.json()

            if not value['statusstring']:
                print "Error"
            elif "FATAL: VoltDB Community Edition" in value['statusstring']:
                print "Voltdb recover is only supported in Enterprise Edition"
            elif "Start request sent successfully to servers" in value['statusstring']:
                self.assertEqual(response.status_code, 200)
                time.sleep(20)
                CheckServerStatus(self, last_db_id, 'running')
                time.sleep(10)
                print "Stopping Cluster...."
                url_stop = 'http://%s:8000/api/1.0/databases/%u/stop' % \
                (__host_or_ip__,last_db_id)
                response = requests.put(url_stop)
                value = response.json()
                if "Server shutdown successfully." in value['statusstring']:
                    self.assertEqual(response.status_code, 200)
                    time.sleep(15)
                    CheckServerStatus(self, last_db_id, 'stopped')
            elif response.status_code == 500:
                self.assertEqual(response.status_code, 500)

def CheckServerStatus(self, last_db_id, status):
    status_url = 'http://%s:8000/api/1.0/databases/%u/status/' % \
    (__host_or_ip__,last_db_id)
    print "Checking status..."
    response = requests.get(status_url)
    value = response.json()
    if value['status'] and value['status'][0]['status']:
        print "Status: " + value['status'][0]['status']
        self.assertEqual(value['status'][0]['status'], status)
        self.assertEqual(value['serverDetails'][0][__host_or_ip__]['status'], status)
    else:
        assert False

if __name__ == '__main__':
    unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports'))
    unittest.main()
<|endoftext|>


<commit_before># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
# OpenFlow 1.2 upgrade, modified for peering exchanges by Sam Russell 2013
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser
from ryu.lib.mac import haddr_to_str

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SPE(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SPE, self).__init__(*args, **kwargs)
        self.mac_to_port = {}

    def add_flow(self, datapath, table_id, match, instructions, priority=0x8000, buffer_id=ofproto_v1_2.OFP_NO_BUFFER):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        #datapath, cookie, cookie_mask, table_id, command,
        #         idle_timeout, hard_timeout, priority, buffer_id, out_port,
        #         out_group, flags, match, instructions)
        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, cookie=0, cookie_mask=0, table_id=table_id,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=0x8000, buffer_id=buffer_id,
            out_port=ofproto_v1_2.OFPP_ANY, out_group=ofproto_v1_2.OFPG_ANY,
            flags=ofproto.OFPFF_SEND_FLOW_REM, match=match, instructions=instructions)
        datapath.send_msg(mod)

    # taken from gre_tunnel.py
    @handler.set_ev_cls(dpset.EventDP)
    def dp_handler(self, ev):
        self.send_event_to_observers(ev)
        enter_leave = ev.enter
        if not enter_leave:
            # TODO:XXX
            # What to do on datapath disconnection?
            self.logger.info('dp disconnection ev:%s', ev)
        else:
            self.logger.info('dp connection ev:%s', ev)

        # will handle ports later

        #dpid = ev.dp.id
        #ports = set(port.port_no for port in ev.ports)
        #ports.update(port.port_no for port in self.nw.get_ports(dpid))
        #for port_no in ports:
        #    self._port_handler(dpid, port_no, enter_leave)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        #iterate through fields - parser should handle this
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        for o in match.fields:
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # do we know the mac?
        if src not in self.mac_to_port[dpid]:
            # learn the mac address to avoid FLOOD next time.
            self.mac_to_port[dpid][src] = in_port
            # set a flow to table 0 to allow packets through to table 1
            match = datapath.ofproto_parser.OFPMatch()
            match.set_in_port(in_port)
            match.set_dl_src(src)
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(1)]
            self.add_flow(datapath, 0, match, instructions)

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            match = datapath.ofproto_parser.OFPMatch()
            match.set_dl_dst(dst)
            instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]
            self.add_flow(datapath, 1, match, instructions, buffer_id=msg.buffer_id)
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            out = datapath.ofproto_parser.OFPPacketOut(
                datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
                actions=actions)
            datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<commit_msg>get namespace right<commit_after># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
# OpenFlow 1.2 upgrade, modified for peering exchanges by Sam Russell 2013
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser
from ryu.lib.mac import haddr_to_str

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SPE(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SPE, self).__init__(*args, **kwargs)
        self.mac_to_port = {}

    def add_flow(self, datapath, table_id, match, instructions, priority=0x8000, buffer_id=ofproto_v1_2.OFP_NO_BUFFER):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        #datapath, cookie, cookie_mask, table_id, command,
        #         idle_timeout, hard_timeout, priority, buffer_id, out_port,
        #         out_group, flags, match, instructions)
        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, cookie=0, cookie_mask=0, table_id=table_id,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=0x8000, buffer_id=buffer_id,
            out_port=ofproto_v1_2.OFPP_ANY, out_group=ofproto_v1_2.OFPG_ANY,
            flags=ofproto.OFPFF_SEND_FLOW_REM, match=match, instructions=instructions)
        datapath.send_msg(mod)

    # taken from gre_tunnel.py
    @set_ev_cls(dpset.EventDP, MAIN_DISPATCHER)
    def dp_handler(self, ev):
        self.send_event_to_observers(ev)
        enter_leave = ev.enter
        if not enter_leave:
            # TODO:XXX
            # What to do on datapath disconnection?
            self.logger.info('dp disconnection ev:%s', ev)
        else:
            self.logger.info('dp connection ev:%s', ev)

        # will handle ports later

        #dpid = ev.dp.id
        #ports = set(port.port_no for port in ev.ports)
        #ports.update(port.port_no for port in self.nw.get_ports(dpid))
        #for port_no in ports:
        #    self._port_handler(dpid, port_no, enter_leave)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        #iterate through fields - parser should handle this
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        for o in match.fields:
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # do we know the mac?
        if src not in self.mac_to_port[dpid]:
            # learn the mac address to avoid FLOOD next time.
            self.mac_to_port[dpid][src] = in_port
            # set a flow to table 0 to allow packets through to table 1
            match = datapath.ofproto_parser.OFPMatch()
            match.set_in_port(in_port)
            match.set_dl_src(src)
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(1)]
            self.add_flow(datapath, 0, match, instructions)

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            match = datapath.ofproto_parser.OFPMatch()
            match.set_dl_dst(dst)
            instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]
            self.add_flow(datapath, 1, match, instructions, buffer_id=msg.buffer_id)
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            out = datapath.ofproto_parser.OFPPacketOut(
                datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
                actions=actions)
            datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<|endoftext|>


<commit_before><commit_msg>Add database models for handling election results.<commit_after>from sqlalchemy import Column, Integer, String, Float, ForeignKey, Index
from sqlalchemy.orm import relationship
from api import db

class VotingDistrict(db.Model):

    __tablename__ = 'voting_districts'

    # columns
    pk = Column(Integer, primary_key=True)
    voting_district_id = Column(Integer, nullable=False)
    year = Column(Integer, nullable=False)
    results_provincial = Column(String)
    results_national = Column(String)
    ward_pk = Column(Integer, ForeignKey('wards.pk'))

    # associations
    ward = relationship("Ward")

    # indexes
    Index('voting_district_year_id_ix', year, voting_district_id, unique=True)

    def __repr__(self):
        return "<VotingDistrict(pk='%s', year='%s', voting_district_id='%s')>" % (
            str(self.pk), str(self.year), str(self.voting_district_id))

class Ward(db.Model):

    __tablename__ = 'wards'

    # columns
    pk = Column(Integer, primary_key=True)
    ward_id = Column(Integer, nullable=False)
    year = Column(Integer, nullable=False)
    results_provincial = Column(String)
    results_national = Column(String)
    municipality_pk = Column(Integer, ForeignKey('municipalities.pk'))

    # associations
    municipality = relationship("Municipality")

    # indexes
    Index('ward_year_id_ix', year, ward_id, unique=True)

    def __repr__(self):
        return "<Ward(pk='%s', year='%s', ward_id='%s')>" % (
            str(self.pk), str(self.year), str(self.ward_id))

class Municipality(db.Model):

    __tablename__ = 'municipalities'

    # columns
    pk = Column(Integer, primary_key=True)
    municipality_id = Column(String, nullable=False)
    year = Column(Integer, nullable=False)
    results_provincial = Column(String)
    results_national = Column(String)
    province_pk = Column(Integer, ForeignKey('provinces.pk'))

    # associations
    province = relationship("Province")

    # indexes
    Index('municipality_year_id_ix', year, municipality_id, unique=True)

    def __repr__(self):
        return "<Municipality(pk='%s', year='%s', municipality_id='%s')>" % (
            str(self.pk), str(self.year), str(self.municipality_id))

class Province(db.Model):

    __tablename__ = 'provinces'

    # columns
    pk = Column(Integer, primary_key=True)
    province_id = Column(String, nullable=False)
    year = Column(Integer, nullable=False)
    results_provincial = Column(String)
    results_national = Column(String)

    # indexes
    Index('province_year_id_ix', year, province_id, unique=True)

    def __repr__(self):
        return "<Province(pk='%s', year='%s', province_id='%s')>" % (
            str(self.pk), str(self.year), str(self.province_id))<|endoftext|>


<commit_before>from __future__ import unicode_literals
import webnotes, os, subprocess, tempfile, json, datetime

@webnotes.whitelist()
def get():
	with open("../app/home/page/latest_updates/latest_updates.json", "r") as lufile:
		return json.loads(lufile.read())

def make():
	def add_to_logs(out, repo):
		out.seek(0)
		last_commit = None
		for l in out.readlines():
			if last_commit is not None:
				if l.startswith("Date:"):
					last_commit["date"] = l[8:-1]
					last_commit["datetime"] = datetime.datetime.strptime(last_commit["date"][:-6], "%a %b %d %H:%M:%S %Y")
				if l.startswith("Author:"):
					last_commit["author"] = l[8:-1]
				if l.startswith("    "):
					last_commit["message"] = l[4:-1]
			if l.startswith("commit"):
				last_commit = {
					"repo": repo,
					"commit": l.split(" ")[1][:-1]
				}
				logs.append(last_commit)

	os.chdir("lib")
	logs = []
	out_lib = tempfile.TemporaryFile()
	subprocess.call("git --no-pager log -n 200", shell=True, stdout=out_lib)
	add_to_logs(out_lib, "lib")

	os.chdir("../app")
	out_app = tempfile.TemporaryFile()
	subprocess.call("git --no-pager log -n 200", shell=True, stdout=out_app)
	add_to_logs(out_app, "app")

	logs.sort(key=lambda a: a["datetime"], reverse=True)
	for a in logs:
		del a["datetime"]

	for i in xrange(len(logs)):
		if i and logs[i]["message"]==logs[i-1]["message"]:
			logs[i]["delete"] = True

		if logs[i]["message"].startswith("Merge branch") or "[" not in logs[i]["message"]:
			logs[i]["delete"] = True

	logs = filter(lambda a: a if not a.get("delete") else None, logs)

	os.chdir("..")
	with open("app/home/page/latest_updates/latest_updates.json", "w") as lufile:
		lufile.write(json.dumps(logs, indent=1, sort_keys=True))

if __name__=="__main__":
	make()<commit_msg>[minor] [fix] use --no-color in latest updates build<commit_after>from __future__ import unicode_literals
import webnotes, os, subprocess, tempfile, json, datetime

@webnotes.whitelist()
def get():
	with open("../app/home/page/latest_updates/latest_updates.json", "r") as lufile:
		return json.loads(lufile.read())

def make():
	def add_to_logs(out, repo):
		out.seek(0)
		last_commit = None
		for l in out.readlines():
			if last_commit is not None:
				if l.startswith("Date:"):
					last_commit["date"] = l[8:-1]
					last_commit["datetime"] = datetime.datetime.strptime(last_commit["date"][:-6], "%a %b %d %H:%M:%S %Y")
				if l.startswith("Author:"):
					last_commit["author"] = l[8:-1]
				if l.startswith("    "):
					last_commit["message"] = l[4:-1]

			if l.startswith("commit"):
				last_commit = {
					"repo": repo,
					"commit": l.split(" ")[1][:-1]
				}
				logs.append(last_commit)

	os.chdir("lib")
	logs = []
	out_lib = tempfile.TemporaryFile()
	subprocess.call("git --no-pager log -n 200 --no-color", shell=True, stdout=out_lib)
	add_to_logs(out_lib, "lib")

	os.chdir("../app")
	out_app = tempfile.TemporaryFile()
	subprocess.call("git --no-pager log -n 200 --no-color", shell=True, stdout=out_app)
	add_to_logs(out_app, "app")

	logs.sort(key=lambda a: a["datetime"], reverse=True)
	for a in logs:
		del a["datetime"]

	for i in xrange(len(logs)):
		if i and logs[i]["message"]==logs[i-1]["message"]:
			logs[i]["delete"] = True

		if logs[i]["message"].startswith("Merge branch") or "[" not in logs[i]["message"]:
			logs[i]["delete"] = True

	logs = filter(lambda a: a if not a.get("delete") else None, logs)

	os.chdir("..")
	with open("app/home/page/latest_updates/latest_updates.json", "w") as lufile:
		lufile.write(json.dumps(logs, indent=1, sort_keys=True))

if __name__=="__main__":
	make()<|endoftext|>


<commit_before>"""

Copyright 2019, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import logging
import json
import django

from flask import request

from django.core.signals import request_finished
from django.contrib.auth.models import User as Django_User
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.conf import settings

from cohorts.models import Cohort_Perms, Cohort, Filters
from accounts.sa_utils import auth_dataset_whitelists_for_user
from cohorts.file_helpers import cohort_files
from cohorts.metadata_helpers import get_sample_case_list
from projects.models import Program

logger = logging.getLogger(settings.LOGGER_NAME)

def validate_user(user_email, cohort_id):
    user = None
    django.setup()
    try:
        user = Django_User.objects.get(email=user_email)
        Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id)
    except ObjectDoesNotExist as e:
        logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e))
    except Exception as e:
        logger.exception(e)

    return user

def get_file_manifest(cohort_id, user):
    file_manifest = None
    inc_filters = {}

    try:
        has_access = auth_dataset_whitelists_for_user(user.id)

        params = {
            'limit': settings.MAX_FILE_LIST_REQUEST,
            'build': 'HG19',
            'access': has_access
        }

        request_data = request.get_json()

        if request_data:
            if request_data['offset']:
                params['offset'] = request.get_assigned_value('offset')

            if request_data['fetch_count']:
                params['limit'] = request.get_assigned_value('fetch_count')

            if request_data['genomic_build']:
                params['build'] = request.get_assigned_value('genomic_build').upper()

            inc_filters = {
                filter: request_data[filter]
                for filter in request_data.keys()
                if filter not in ['cohort_id', 'fetch_count', 'offset', 'genomic_build']
            }

        response = cohort_files(cohort_id, user=user, inc_filters=inc_filters, **params)

        file_manifest = response['file_list'] if response and response['file_list'] else None

    except Exception as e:
        logger.exception(e)

    return file_manifest

def get_cohort_info(cohort_id):
    cohort = None
    try:
        cohort_obj = Cohort.objects.get(id=cohort_id)
        cohort = {
            'id': cohort_obj.id,
            'name': cohort_obj.name,
            'case_count': cohort_obj.case_size(),
            'sample_count': cohort_obj.sample_size(),
            'programs': cohort_obj.get_program_names()
        }
    except ObjectDoesNotExist as e:
        logger.warn("Cohort with ID {} was not found!".format(str(cohort_id)))
    except Exception as e:
        logger.exception(e)

    return cohort

def get_cohorts(user_email):

    cohort_list = None

    try:
        user = Django_User.objects.get(email=user_email)
        cohort_perms = Cohort_Perms.objects.filter(user_id=user.id)
        cohort_list = []
        for cohort_perm in cohort_perms:
            cohort_list.append({
                'id': cohort_perm.cohort.id,
                'name': cohort_perm.cohort.name,
                'permission': cohort_perm.perm
            })

    except ObjectDoesNotExist as e:
        logger.info("No cohorts found for user {}".format(user_email))

    return cohort_list

def get_cohort_counts():
    cohort_counts = None

    try:
        request_data = request.get_json()
        print("Data: " + str(request_data))
        for prog_name in request_data['filters']:
            try:
                prog_filters = None
                this_program = Program.objects.get(name=prog_name.upper(), is_public=1, active=1)
                if request_data['filters'][prog_name]:
                    prog_filters = request_data['filters'][prog_name]
                prog_counts = get_sample_case_list(None, prog_filters, None, this_program.id)
                if prog_counts:
                    if not cohort_counts:
                        cohort_counts = {}
                    cohort_counts[prog_name] = prog_counts
                else:
                    logger.warn("Could not obtain program counts for {} - skipping.".format(
                        prog_name
                    ))
            except ObjectDoesNotExist as e:
                logger.error("Program {} was not found as an active, public program - skipping.".format(
                    prog_name
                ))
                continue
    except Exception as e:
        logger.exception(e)

    return cohort_counts

<commit_msg>-> Swap to new case list builder<commit_after>"""

Copyright 2019, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import logging
import json
import django

from flask import request

from django.core.signals import request_finished
from django.contrib.auth.models import User as Django_User
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.conf import settings

from cohorts.models import Cohort_Perms, Cohort, Filters
from accounts.sa_utils import auth_dataset_whitelists_for_user
from cohorts.file_helpers import cohort_files
from cohorts.metadata_helpers import get_sample_case_list_bq
from projects.models import Program

logger = logging.getLogger(settings.LOGGER_NAME)

def validate_user(user_email, cohort_id):
    user = None
    django.setup()
    try:
        user = Django_User.objects.get(email=user_email)
        Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id)
    except ObjectDoesNotExist as e:
        logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e))
    except Exception as e:
        logger.exception(e)

    return user

def get_file_manifest(cohort_id, user):
    file_manifest = None
    inc_filters = {}

    try:
        has_access = auth_dataset_whitelists_for_user(user.id)

        params = {
            'limit': settings.MAX_FILE_LIST_REQUEST,
            'build': 'HG19',
            'access': has_access
        }

        request_data = request.get_json()

        if request_data:
            if request_data['offset']:
                params['offset'] = request.get_assigned_value('offset')

            if request_data['fetch_count']:
                params['limit'] = request.get_assigned_value('fetch_count')

            if request_data['genomic_build']:
                params['build'] = request.get_assigned_value('genomic_build').upper()

            inc_filters = {
                filter: request_data[filter]
                for filter in request_data.keys()
                if filter not in ['cohort_id', 'fetch_count', 'offset', 'genomic_build']
            }

        response = cohort_files(cohort_id, user=user, inc_filters=inc_filters, **params)

        file_manifest = response['file_list'] if response and response['file_list'] else None

    except Exception as e:
        logger.exception(e)

    return file_manifest

def get_cohort_info(cohort_id):
    cohort = None
    try:
        cohort_obj = Cohort.objects.get(id=cohort_id)
        cohort = {
            'id': cohort_obj.id,
            'name': cohort_obj.name,
            'case_count': cohort_obj.case_size(),
            'sample_count': cohort_obj.sample_size(),
            'programs': cohort_obj.get_program_names()
        }
    except ObjectDoesNotExist as e:
        logger.warn("Cohort with ID {} was not found!".format(str(cohort_id)))
    except Exception as e:
        logger.exception(e)

    return cohort

def get_cohorts(user_email):

    cohort_list = None

    try:
        user = Django_User.objects.get(email=user_email)
        cohort_perms = Cohort_Perms.objects.filter(user_id=user.id)
        cohort_list = []
        for cohort_perm in cohort_perms:
            cohort_list.append({
                'id': cohort_perm.cohort.id,
                'name': cohort_perm.cohort.name,
                'permission': cohort_perm.perm
            })

    except ObjectDoesNotExist as e:
        logger.info("No cohorts found for user {}".format(user_email))

    return cohort_list

def get_cohort_counts():
    cohort_counts = None

    try:
        request_data = request.get_json()
        cohort_counts = get_sample_case_list_bq(None, request_data['filters'])
    except Exception as e:
        logger.exception(e)

    return cohort_counts

<|endoftext|>


<commit_before><commit_msg>DataAccess.py now uses the QSDATA environmental variable to find data<commit_after><|endoftext|>